geonames_dump 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +96 -47
- data/lib/generators/geonames_dump/install_generator.rb +4 -4
- data/lib/generators/geonames_dump/templates/app/models/geonames_alternate_name.rb +52 -0
- data/lib/generators/geonames_dump/templates/app/models/geonames_feature.rb +9 -2
- data/lib/generators/geonames_dump/templates/db/migrate/create_geonames_alternate_names.rb +32 -0
- data/lib/generators/geonames_dump/templates/db/migrate/create_geonames_countries.rb +2 -10
- data/lib/generators/geonames_dump/templates/db/migrate/create_geonames_features.rb +20 -25
- data/lib/generators/geonames_dump/templates/db/migrate/create_geonames_iso_languagecodes.rb +17 -0
- data/lib/geonames_dump.rb +31 -0
- data/lib/geonames_dump/version.rb +1 -1
- data/lib/tasks/import.rake +37 -17
- data/lib/tasks/install.rake +1 -1
- data/lib/tasks/truncate.rake +7 -4
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 449ac0186316849284960960df5200fa5830053c
|
4
|
+
data.tar.gz: 647d2b017b3c9cdc5ef44c4fde827e74b3492ebd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: df1f520a053206501c209a3b23a9b1cd9385ac18dbc05d91f3176c533dd355853b733aa066e3f9d1afd6f07da856fc2d5a3455797d5027bfe0fd123e939591ef
|
7
|
+
data.tar.gz: 1e3a7f6ed3f2b63dcc83565a532fdad2bb4cb3e70a56006c87e9097a963cedb61e5e26bbc9491cfc6caa37289f54652460a0990e4dbd3dfafdf340594da34dbe
|
data/README.md
CHANGED
@@ -1,80 +1,129 @@
|
|
1
1
|
# GeonamesDump
|
2
2
|
|
3
|
-
GeonamesDump import geographic data from geonames project into your
|
4
|
-
It's a "gem"
|
5
|
-
|
3
|
+
GeonamesDump import geographic data from geonames project into your
|
4
|
+
application, avoiding to use external service like google maps. It's a "gem"
|
5
|
+
version of the
|
6
|
+
application [brownbeagle/geonames](https://github.com/brownbeagle/geonames).
|
7
|
+
Now you only need to include the dependency into your Gemfile and your project
|
8
|
+
will include geonames.
|
6
9
|
|
7
|
-
You're free to use [geocoder](https://github.com/alexreisner/geocoder) or
|
10
|
+
You're free to use [geocoder](https://github.com/alexreisner/geocoder) or
|
11
|
+
[geokit](https://github.com/imajes/geokit) or any other geocoding solution.
|
8
12
|
|
9
13
|
## Installation
|
10
14
|
|
11
15
|
Add this line to your application's Gemfile:
|
12
16
|
|
13
|
-
|
17
|
+
```
|
18
|
+
gem 'geonames_dump'
|
19
|
+
```
|
14
20
|
|
15
21
|
And then execute:
|
16
22
|
|
17
|
-
|
23
|
+
```
|
24
|
+
bundle install
|
25
|
+
```
|
18
26
|
|
19
27
|
Or install it yourself as:
|
20
28
|
|
21
|
-
|
29
|
+
```
|
30
|
+
gem install geonames_dump
|
31
|
+
```
|
22
32
|
|
23
33
|
## Usage
|
24
34
|
|
25
35
|
Create models and migration files
|
26
36
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
37
|
+
```
|
38
|
+
rails generate geonames_dump:install
|
39
|
+
```
|
40
|
+
|
41
|
+
Import data (takes a loonnnng time!), it will download data, import countries
|
42
|
+
and many features (Countries, Cities having more than 15000 people, Admin1
|
43
|
+
(first administrative subdivision), Admin2 (second level administrative
|
44
|
+
subdivision))
|
45
|
+
|
46
|
+
```
|
47
|
+
rake geonames_dump:install
|
48
|
+
```
|
49
|
+
|
50
|
+
If you need more fine grained control over the installation process you can run
|
51
|
+
individual geoname rake tasks instead of the all-in-one install :
|
52
|
+
|
53
|
+
```
|
54
|
+
$ rake -T | grep geonames_dump
|
55
|
+
|
56
|
+
rake geonames_dump:import:all # Import ALL geonames data.
|
57
|
+
rake geonames_dump:import:many # Import most of geonames data.
|
58
|
+
|
59
|
+
rake geonames_dump:import:admin1 # Import admin1 codes
|
60
|
+
rake geonames_dump:import:admin2 # Import admin2 codes
|
61
|
+
rake geonames_dump:import:cities # Import all cities, regardless of population.
|
62
|
+
rake geonames_dump:import:cities1000 # Import cities with population greater than 1000
|
63
|
+
rake geonames_dump:import:cities15000 # Import cities with population greater than 15000
|
64
|
+
rake geonames_dump:import:cities5000 # Import cities with population greater than 5000
|
65
|
+
rake geonames_dump:import:countries # Import countries informations
|
66
|
+
rake geonames_dump:import:features # Import feature data.
|
67
|
+
rake geonames_dump:import:alternate_names # Import alternate names
|
68
|
+
|
69
|
+
rake geonames_dump:truncate:all # Truncate all geonames data.
|
70
|
+
rake geonames_dump:truncate:countries # Truncate countries informations
|
71
|
+
rake geonames_dump:truncate:admin1 # Truncate admin1 codes
|
72
|
+
rake geonames_dump:truncate:admin2 # Truncate admin2 codes
|
73
|
+
rake geonames_dump:truncate:cities # Truncate cities informations
|
74
|
+
rake geonames_dump:truncate:features # Truncate features informations
|
75
|
+
rake geonames_dump:truncate:alternate_names # Import alternate names
|
76
|
+
```
|
55
77
|
|
56
78
|
## Geonames data usage
|
57
79
|
|
58
|
-
The above commands will import geonames data in your Rails application, in
|
80
|
+
The above commands will import geonames data in your Rails application, in
|
81
|
+
other words, this will create models and fill database with place/city/country
|
82
|
+
informations.
|
83
|
+
|
84
|
+
A convenient way to search for data is to use GeonamesDump search accessor
|
85
|
+
`GeonamesDump.search`. This method interate on data types to find a result.
|
86
|
+
Search order is the following :
|
87
|
+
|
88
|
+
1. Cities
|
89
|
+
2. Alternate names (names in non-latin alphabets)
|
90
|
+
3. First level admin subdivisions
|
91
|
+
4. Second level admin subdivisions
|
92
|
+
5. Countries
|
93
|
+
6. Features (lakes, montains and others various features)
|
94
|
+
|
59
95
|
Now to find a city for example :
|
60
96
|
|
61
|
-
|
97
|
+
```
|
98
|
+
GeonamesDump.search('paris')
|
99
|
+
```
|
62
100
|
|
63
|
-
If your request is ambiguous, like not searching Dublin in Ireland but Dublin
|
101
|
+
If your request is ambiguous, like not searching Dublin in Ireland but Dublin
|
102
|
+
in the USA, you may specify country :
|
64
103
|
|
65
|
-
|
104
|
+
```
|
105
|
+
GeonamesDump.search('dublin').first.country_code
|
106
|
+
=> 'IE'
|
107
|
+
GeonamesDump.search('dublin, us').first.country_code
|
108
|
+
=> 'US'
|
109
|
+
```
|
66
110
|
|
67
|
-
|
111
|
+
If needed, requested type may be specified too :
|
68
112
|
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
113
|
+
```
|
114
|
+
GeonamesDump.search('dublin', type: :city)
|
115
|
+
GeonamesDump.search('dublin, us', type: :city)
|
116
|
+
GeonamesDump.search('paris', type: :feature)
|
117
|
+
```
|
74
118
|
|
75
|
-
|
119
|
+
The following types are available :
|
76
120
|
|
77
|
-
|
121
|
+
- admin1, for first level of adminstrative subdivision
|
122
|
+
- admin2, for second level of adminstrative subdivision
|
123
|
+
- city, for city names
|
124
|
+
- feature, for generic names including all the above
|
125
|
+
- alternate_name, for names in non-latin alphabets
|
126
|
+
- country, for country names
|
78
127
|
|
79
128
|
## Contributing
|
80
129
|
|
@@ -9,15 +9,15 @@ Description:
|
|
9
9
|
Create Geonames initial migrations files and Geonames model files in your application
|
10
10
|
DESC
|
11
11
|
def copy_migrations_files
|
12
|
-
|
13
|
-
|
12
|
+
Dir.glob(File.join(File.expand_path(File.join('..', 'templates', 'db', 'migrate'), __FILE__), '*')).each do |full_path|
|
13
|
+
file = File.basename(full_path, File.extname(full_path))
|
14
14
|
migration_template File.join('db', 'migrate', "#{file}.rb")
|
15
15
|
end
|
16
16
|
end
|
17
17
|
|
18
18
|
def copy_models_files
|
19
|
-
|
20
|
-
|
19
|
+
Dir.glob(File.join(File.expand_path(File.join('..', 'templates', 'app', 'models'), __FILE__), '*')).each do |full_path|
|
20
|
+
file = File.basename(full_path, File.extname(full_path))
|
21
21
|
copy_file File.join('app', 'models', "#{file}.rb")
|
22
22
|
end
|
23
23
|
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
class GeonamesAlternateName < ActiveRecord::Base
|
2
|
+
validates_uniqueness_of :alternate_name_id
|
3
|
+
validates_uniqueness_of :geonameid
|
4
|
+
before_save :set_alternate_name_first_letters
|
5
|
+
|
6
|
+
##
|
7
|
+
# default search (by alternate name)
|
8
|
+
#
|
9
|
+
scope :search, lambda { |q|
|
10
|
+
by_alternate_name(q)
|
11
|
+
}
|
12
|
+
|
13
|
+
##
|
14
|
+
# search by isolanguage code
|
15
|
+
#
|
16
|
+
scope :by_isolanguage, lambda { |q|
|
17
|
+
where("isolanguage = ?", q)
|
18
|
+
}
|
19
|
+
|
20
|
+
##
|
21
|
+
# search prefered names
|
22
|
+
#
|
23
|
+
scope :prefered, lambda {
|
24
|
+
where(is_preferred_name: true)
|
25
|
+
}
|
26
|
+
|
27
|
+
##
|
28
|
+
# search by name
|
29
|
+
#
|
30
|
+
scope :by_alternate_name, lambda { |q|
|
31
|
+
ret = self.scoped
|
32
|
+
ret = ret.where("alternate_name_first_letters = ?", q[0...3].downcase)
|
33
|
+
ret = ret.where("alternate_name LIKE ?", "#{q}%")
|
34
|
+
}
|
35
|
+
|
36
|
+
##
|
37
|
+
# Get associated feature
|
38
|
+
#
|
39
|
+
def feature
|
40
|
+
GeonamesFeature.where(geonameid: self.geonameid)
|
41
|
+
end
|
42
|
+
|
43
|
+
protected
|
44
|
+
|
45
|
+
##
|
46
|
+
# Set first letters of name into index column
|
47
|
+
#
|
48
|
+
def set_alternate_name_first_letters
|
49
|
+
self.alternate_name_first_letters = self.alternate_name[0...3].downcase unless self.alternate_name.nil?
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
@@ -5,7 +5,7 @@ class GeonamesFeature < ActiveRecord::Base
|
|
5
5
|
##
|
6
6
|
# Search for feature, searches might include country (separated by ',')
|
7
7
|
#
|
8
|
-
scope :search, lambda { |query|
|
8
|
+
scope :search, lambda { |query|
|
9
9
|
virgule = query.include? ','
|
10
10
|
|
11
11
|
splited = virgule ? query.split(',') : [query]
|
@@ -19,7 +19,7 @@ class GeonamesFeature < ActiveRecord::Base
|
|
19
19
|
|
20
20
|
unless country.nil?
|
21
21
|
geonames_country = GeonamesCountry.search(country).first
|
22
|
-
ret = ret.where(:
|
22
|
+
ret = ret.where(:country_code => geonames_country.iso) unless geonames_country.nil?
|
23
23
|
end
|
24
24
|
|
25
25
|
ret
|
@@ -39,6 +39,13 @@ class GeonamesFeature < ActiveRecord::Base
|
|
39
39
|
ret
|
40
40
|
}
|
41
41
|
|
42
|
+
def alternate_names
|
43
|
+
GeonamesAlternateName.where(geonameid: self.geonameid)
|
44
|
+
end
|
45
|
+
|
46
|
+
|
47
|
+
protected
|
48
|
+
|
42
49
|
##
|
43
50
|
# Set first letters of name into index column
|
44
51
|
#
|
@@ -0,0 +1,32 @@
|
|
1
|
+
class CreateGeonamesAlternateNames < ActiveRecord::Migration
|
2
|
+
# http://download.geonames.org/export/dump/countryInfo.txt
|
3
|
+
# alternateNameId : the id of this alternate name, int
|
4
|
+
# geonameid : geonameId referring to id in table 'geoname', int
|
5
|
+
# isolanguage : iso 639 language code 2- or 3-characters; 4-characters 'post' for postal codes and 'iata','icao' and faac for airport codes, fr_1793 for French Revolution names, abbr for abbreviation, link for a website, varchar(7)
|
6
|
+
# alternate name : alternate name or name variant, varchar(200)
|
7
|
+
# isPreferredName : '1', if this alternate name is an official/preferred name
|
8
|
+
# isShortName : '1', if this is a short name like 'California' for 'State of California'
|
9
|
+
# isColloquial : '1', if this alternate name is a colloquial or slang term
|
10
|
+
# isHistoric : '1', if this alternate name is historic and was used in the past
|
11
|
+
def change
|
12
|
+
create_table :geonames_alternate_names do |t|
|
13
|
+
t.integer :alternate_name_id
|
14
|
+
t.integer :geonameid
|
15
|
+
t.string :isolanguage, length: 7
|
16
|
+
t.string :alternate_name, length: 200
|
17
|
+
t.boolean :is_preferred_name
|
18
|
+
t.boolean :is_short_name
|
19
|
+
t.boolean :is_colloquial
|
20
|
+
t.boolean :is_historic
|
21
|
+
|
22
|
+
t.string :alternate_name_first_letters
|
23
|
+
|
24
|
+
t.timestamps
|
25
|
+
end
|
26
|
+
|
27
|
+
add_index :geonames_alternate_names, :alternate_name_id
|
28
|
+
add_index :geonames_alternate_names, :geonameid
|
29
|
+
add_index :geonames_alternate_names, :isolanguage
|
30
|
+
add_index :geonames_alternate_names, :alternate_name_first_letters
|
31
|
+
end
|
32
|
+
end
|
@@ -1,9 +1,6 @@
|
|
1
|
-
|
2
|
-
# http://download.geonames.org/export/dump/countryInfo.txt
|
3
|
-
|
4
1
|
class CreateGeonamesCountries < ActiveRecord::Migration
|
5
|
-
|
6
|
-
def
|
2
|
+
# http://download.geonames.org/export/dump/countryInfo.txt
|
3
|
+
def change
|
7
4
|
create_table :geonames_countries do |t|
|
8
5
|
t.string :iso
|
9
6
|
t.string :iso3
|
@@ -29,10 +26,5 @@ class CreateGeonamesCountries < ActiveRecord::Migration
|
|
29
26
|
end
|
30
27
|
|
31
28
|
add_index :geonames_countries, :geonameid
|
32
|
-
add_index :geonames_countries, :country
|
33
|
-
end
|
34
|
-
|
35
|
-
def self.down
|
36
|
-
drop_table :geonames_countries
|
37
29
|
end
|
38
30
|
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
class CreateGeonamesFeatures < ActiveRecord::Migration
|
2
|
-
|
2
|
+
# http://download.geonames.org/export/dump/readme.txt
|
3
3
|
# geonameid : integer id of record in geonames database
|
4
4
|
# name : name of geographical point (utf8) varchar(200)
|
5
5
|
# asciiname : name of geographical point in plain ascii characters, varchar(200)
|
@@ -11,54 +11,49 @@ class CreateGeonamesFeatures < ActiveRecord::Migration
|
|
11
11
|
# country code : ISO-3166 2-letter country code, 2 characters
|
12
12
|
# cc2 : alternate country codes, comma separated, ISO-3166 2-letter country code, 60 characters
|
13
13
|
# admin1 code : fipscode (subject to change to iso code), isocode for the us and ch, see file admin1Codes.txt for display names of this code; varchar(20)
|
14
|
-
# admin2 code : code for the second administrative division, a county in the US, see file admin2Codes.txt; varchar(80)
|
14
|
+
# admin2 code : code for the second administrative division, a county in the US, see file admin2Codes.txt; varchar(80)
|
15
15
|
# admin3 code : code for third level administrative division, varchar(20)
|
16
16
|
# admin4 code : code for fourth level administrative division, varchar(20)
|
17
|
-
# population : bigint (4 byte int)
|
17
|
+
# population : bigint (4 byte int)
|
18
18
|
# elevation : in meters, integer
|
19
|
-
#
|
19
|
+
# dem : digital elevation model, srtm3 or gtopo30, average elevation of 3''x3'' (ca 90mx90m) or 30''x30'' (ca 900mx900m) area in meters, integer. srtm processed by cgiar/ciat.
|
20
20
|
# timezone : the timezone id (see file timeZone.txt)
|
21
21
|
# modification date : date of last modification in yyyy-MM-dd format
|
22
|
-
def
|
22
|
+
def change
|
23
23
|
create_table :geonames_features do |t|
|
24
24
|
t.integer :geonameid
|
25
|
-
t.string :name
|
26
|
-
t.string :asciiname
|
27
|
-
t.string :alternatenames
|
25
|
+
t.string :name, length: 200
|
26
|
+
t.string :asciiname, length: 200
|
27
|
+
t.string :alternatenames, length: 5000
|
28
28
|
t.float :latitude
|
29
29
|
t.float :longitude
|
30
30
|
t.string :feature_class
|
31
|
-
t.string :
|
32
|
-
t.string :
|
33
|
-
t.string :cc2
|
34
|
-
t.string :
|
35
|
-
t.string :
|
36
|
-
t.string :
|
37
|
-
t.string :
|
31
|
+
t.string :feature_code, length: 10
|
32
|
+
t.string :country_code
|
33
|
+
t.string :cc2, length: 60
|
34
|
+
t.string :admin1_code, length: 20
|
35
|
+
t.string :admin2_code, length: 80
|
36
|
+
t.string :admin3_code, length: 20
|
37
|
+
t.string :admin4_code, length: 20
|
38
38
|
t.integer :population
|
39
39
|
t.integer :elevation
|
40
|
-
t.integer :
|
41
|
-
t.string :timezone
|
40
|
+
t.integer :dem
|
41
|
+
t.string :timezone, length: 40
|
42
42
|
t.timestamp :modification
|
43
|
+
|
43
44
|
t.string :type
|
44
|
-
# OPTIMIZE: create another table for all names (alternate names)
|
45
45
|
t.string :asciiname_first_letters
|
46
46
|
|
47
|
-
|
48
47
|
t.timestamps
|
49
48
|
end
|
50
49
|
|
51
50
|
add_index :geonames_features, :geonameid
|
52
51
|
add_index :geonames_features, :name
|
53
52
|
add_index :geonames_features, :asciiname
|
54
|
-
add_index :geonames_features, :
|
53
|
+
add_index :geonames_features, :country_code
|
55
54
|
add_index :geonames_features, :population
|
56
|
-
add_index :geonames_features, :
|
55
|
+
add_index :geonames_features, :admin1_code
|
57
56
|
add_index :geonames_features, :type
|
58
57
|
add_index :geonames_features, :asciiname_first_letters
|
59
58
|
end
|
60
|
-
|
61
|
-
def self.down
|
62
|
-
drop_table :geonames_features
|
63
|
-
end
|
64
59
|
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
class CreateGeonamesIsoLanguagecodes < ActiveRecord::Migration
|
2
|
+
# http://download.geonames.org/export/dump/iso-languagecodes.txt
|
3
|
+
# iso_639_3 : ISO 639-3 language code
|
4
|
+
# iso_639_2 : ISO 639-2 language code
|
5
|
+
# iso_639_1 : ISO 639-1 language code
|
6
|
+
# language_name : language name
|
7
|
+
def change
|
8
|
+
create_table :geonames_iso_languagecodes do |t|
|
9
|
+
t.string :iso_639_3, length: 4
|
10
|
+
t.string :iso_639_2, length: 4
|
11
|
+
t.string :iso_639_1, length: 4
|
12
|
+
t.string :language_name, length: 80
|
13
|
+
|
14
|
+
t.timestamps
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
data/lib/geonames_dump.rb
CHANGED
@@ -5,4 +5,35 @@ require "geonames_dump/railtie" #if defined?(Rails)
|
|
5
5
|
module GeonamesDump
|
6
6
|
# TODO: remove this logger
|
7
7
|
#ActiveRecord::Base.logger = Logger.new(STDOUT) if Rails.env.development? && !ENV['SILENT']
|
8
|
+
|
9
|
+
def self.search(query, options = {})
|
10
|
+
ret = nil
|
11
|
+
|
12
|
+
type = options[:type] || :auto
|
13
|
+
begin
|
14
|
+
case type
|
15
|
+
when :auto
|
16
|
+
# city name
|
17
|
+
ret = GeonamesCity.search(query)
|
18
|
+
# alternate name
|
19
|
+
ret = GeonamesAlternateName.search(query) if ret.blank?
|
20
|
+
# admin1
|
21
|
+
ret = GeonamesAdmin1.search(query) if ret.blank?
|
22
|
+
# admin2
|
23
|
+
ret = GeonamesAdmin2.search(query) if ret.blank?
|
24
|
+
# country
|
25
|
+
ret = GeonamesCountry.search(query) if ret.blank?
|
26
|
+
# feature
|
27
|
+
ret = GeonamesFeature.search(query) if ret.blank?
|
28
|
+
else
|
29
|
+
model = "geonames_#{type.to_s}".camelcase.constantize
|
30
|
+
ret = model.search(query)
|
31
|
+
end
|
32
|
+
rescue NameError => e
|
33
|
+
Rails.logger.error "Unknown type for GeonamesDump : #{e}"
|
34
|
+
#raise
|
35
|
+
end
|
36
|
+
|
37
|
+
ret
|
38
|
+
end
|
8
39
|
end
|
data/lib/tasks/import.rake
CHANGED
@@ -8,9 +8,14 @@ namespace :geonames_dump do
|
|
8
8
|
CACHE_DIR = "#{Rails.root}/db/geonames_cache"
|
9
9
|
|
10
10
|
GEONAMES_FEATURES_COL_NAME = [
|
11
|
-
:geonameid, :name, :asciiname, :alternatenames, :latitude, :longitude,
|
12
|
-
:
|
13
|
-
:
|
11
|
+
:geonameid, :name, :asciiname, :alternatenames, :latitude, :longitude,
|
12
|
+
:feature_class, :feature_code, :country_code, :cc2, :admin1_code,
|
13
|
+
:admin2_code, :admin3_code, :admin4_code, :population, :elevation,
|
14
|
+
:dem, :timezone, :modification
|
15
|
+
]
|
16
|
+
GEONAMES_ALTERNATE_NAMES_COL_NAME = [
|
17
|
+
:alternate_name_id, :geonameid, :isolanguage, :alternate_name,
|
18
|
+
:is_preferred_name, :is_short_name, :is_colloquial, :is_historic
|
14
19
|
]
|
15
20
|
GEONAMES_COUNTRIES_COL_NAME = [
|
16
21
|
:iso, :iso3, :iso_numeric, :fips, :country, :capital, :area, :population, :continent,
|
@@ -30,7 +35,7 @@ namespace :geonames_dump do
|
|
30
35
|
task :all => [:many, :features]
|
31
36
|
|
32
37
|
desc 'Import most of geonames data. Recommended after a clean install.'
|
33
|
-
task :many => [:prepare, :countries, :
|
38
|
+
task :many => [:prepare, :countries, :cities15000, :admin1, :admin2]
|
34
39
|
|
35
40
|
desc 'Import all cities, regardless of population.'
|
36
41
|
task :cities => [:prepare, :cities15000, :cities5000, :cities1000]
|
@@ -44,8 +49,6 @@ namespace :geonames_dump do
|
|
44
49
|
|
45
50
|
# Import into the database.
|
46
51
|
File.open(txt_file) do |f|
|
47
|
-
#VALID_FEATURES = %w[ADMIN1]
|
48
|
-
#feature_attrs = VALID_FEATURES & ENV.keys
|
49
52
|
# TODO: add feature selection
|
50
53
|
insert_data(f, GEONAMES_FEATURES_COL_NAME, GeonamesCity, :title => "Features")
|
51
54
|
end
|
@@ -58,7 +61,6 @@ namespace :geonames_dump do
|
|
58
61
|
|
59
62
|
txt_file = get_or_download("http://download.geonames.org/export/dump/cities#{population}.zip")
|
60
63
|
|
61
|
-
# Import into the database.
|
62
64
|
File.open(txt_file) do |f|
|
63
65
|
insert_data(f, GEONAMES_FEATURES_COL_NAME, GeonamesCity, :title => "cities of #{population}")
|
64
66
|
end
|
@@ -66,10 +68,29 @@ namespace :geonames_dump do
|
|
66
68
|
end
|
67
69
|
|
68
70
|
desc 'Import countries informations'
|
69
|
-
task :countries => :environment do
|
71
|
+
task :countries => [:prepare, :environment] do
|
70
72
|
txt_file = get_or_download('http://download.geonames.org/export/dump/countryInfo.txt')
|
71
73
|
|
72
|
-
|
74
|
+
File.open(txt_file) do |f|
|
75
|
+
insert_data(f, GEONAMES_COUNTRIES_COL_NAME, GeonamesCountry, :title => "Countries")
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
desc 'Import alternate names'
|
80
|
+
task :alternate_names => [:prepare, :environment] do
|
81
|
+
txt_file = get_or_download('http://download.geonames.org/export/dump/alternateNames.zip',
|
82
|
+
txt_file: 'alternateNames.txt')
|
83
|
+
|
84
|
+
File.open(txt_file) do |f|
|
85
|
+
insert_data(f, GEONAMES_ALTERNATE_NAMES_COL_NAME, GeonamesAlternateName, :title => "Alternate names", :buffer => 10000)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
desc 'Import iso language codes'
|
90
|
+
task :language_codes => [:prepare, :environment] do
|
91
|
+
txt_file = get_or_download('http://download.geonames.org/export/dump/alternateNames.zip',
|
92
|
+
txt_file: 'iso-languagecodes.txt')
|
93
|
+
|
73
94
|
File.open(txt_file) do |f|
|
74
95
|
insert_data(f, GEONAMES_COUNTRIES_COL_NAME, GeonamesCountry, :title => "Countries")
|
75
96
|
end
|
@@ -79,15 +100,14 @@ namespace :geonames_dump do
|
|
79
100
|
task :admin1 => [:prepare, :environment] do
|
80
101
|
txt_file = get_or_download('http://download.geonames.org/export/dump/admin1CodesASCII.txt')
|
81
102
|
|
82
|
-
# Import into the database.
|
83
103
|
File.open(txt_file) do |f|
|
84
104
|
insert_data(f, GEONAMES_ADMINS_COL_NAME, GeonamesAdmin1, :title => "Admin1 subdivisions") do |klass, attributes, col_value, idx|
|
85
105
|
col_value.gsub!('(general)', '')
|
86
106
|
col_value.strip!
|
87
107
|
if idx == 0
|
88
108
|
country, admin1 = col_value.split('.')
|
89
|
-
attributes[:
|
90
|
-
attributes[:
|
109
|
+
attributes[:country_code] = country.strip
|
110
|
+
attributes[:admin1_code] = admin1.strip rescue nil
|
91
111
|
else
|
92
112
|
attributes[GEONAMES_ADMINS_COL_NAME[idx]] = col_value
|
93
113
|
end
|
@@ -99,15 +119,14 @@ namespace :geonames_dump do
|
|
99
119
|
task :admin2 => [:prepare, :environment] do
|
100
120
|
txt_file = get_or_download('http://download.geonames.org/export/dump/admin2Codes.txt')
|
101
121
|
|
102
|
-
# Import into the database.
|
103
122
|
File.open(txt_file) do |f|
|
104
123
|
insert_data(f, GEONAMES_ADMINS_COL_NAME, GeonamesAdmin2, :title => "Admin2 subdivisions") do |klass, attributes, col_value, idx|
|
105
124
|
col_value.gsub!('(general)', '')
|
106
125
|
if idx == 0
|
107
126
|
country, admin1, admin2 = col_value.split('.')
|
108
|
-
attributes[:
|
109
|
-
attributes[:
|
110
|
-
attributes[:
|
127
|
+
attributes[:country_code] = country.strip
|
128
|
+
attributes[:admin1_code] = admin1.strip #rescue nil
|
129
|
+
attributes[:admin2_code] = admin2.strip #rescue nil
|
111
130
|
else
|
112
131
|
attributes[GEONAMES_ADMINS_COL_NAME[idx]] = col_value
|
113
132
|
end
|
@@ -179,6 +198,7 @@ namespace :geonames_dump do
|
|
179
198
|
# Setup nice progress output.
|
180
199
|
file_size = file_fd.stat.size
|
181
200
|
title = options[:title] || 'Feature Import'
|
201
|
+
buffer = options[:buffer] || 1000
|
182
202
|
progress_bar = ProgressBar.create(:title => title, :total => file_size, :format => '%a |%b>%i| %p%% %t')
|
183
203
|
|
184
204
|
# create block array
|
@@ -223,7 +243,7 @@ namespace :geonames_dump do
|
|
223
243
|
end
|
224
244
|
|
225
245
|
# increase import speed by performing insert using transaction
|
226
|
-
if line_counter %
|
246
|
+
if line_counter % buffer == 0
|
227
247
|
ActiveRecord::Base.transaction do
|
228
248
|
blocks.call_and_reset
|
229
249
|
end
|
data/lib/tasks/install.rake
CHANGED
data/lib/tasks/truncate.rake
CHANGED
@@ -23,18 +23,21 @@ namespace :geonames_dump do
|
|
23
23
|
task :cities => :environment do
|
24
24
|
GeonamesCity.delete_all #&& GeonamesCity.reset_pk_sequence
|
25
25
|
end
|
26
|
-
|
26
|
+
|
27
27
|
desc 'Truncate countries informations'
|
28
28
|
task :countries => :environment do
|
29
29
|
GeonamesCountry.delete_all && GeonamesCountry.reset_pk_sequence
|
30
30
|
end
|
31
|
-
|
31
|
+
|
32
32
|
desc 'Truncate features informations'
|
33
33
|
task :features => :environment do
|
34
34
|
GeonamesFeature.delete_all && GeonamesFeature.reset_pk_sequence
|
35
35
|
end
|
36
|
-
|
37
|
-
|
36
|
+
|
37
|
+
desc 'Truncate alternate names'
|
38
|
+
task :alternate_names => :environment do
|
39
|
+
GeonamesAlternateName.delete_all && GeonamesAlternateName.reset_pk_sequence
|
40
|
+
end
|
38
41
|
|
39
42
|
end
|
40
43
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: geonames_dump
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alex Pooley
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-11-
|
12
|
+
date: 2013-11-26 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: ruby-progressbar
|
@@ -70,11 +70,14 @@ files:
|
|
70
70
|
- lib/generators/geonames_dump/install_generator.rb
|
71
71
|
- lib/generators/geonames_dump/templates/app/models/geonames_admin1.rb
|
72
72
|
- lib/generators/geonames_dump/templates/app/models/geonames_admin2.rb
|
73
|
+
- lib/generators/geonames_dump/templates/app/models/geonames_alternate_name.rb
|
73
74
|
- lib/generators/geonames_dump/templates/app/models/geonames_city.rb
|
74
75
|
- lib/generators/geonames_dump/templates/app/models/geonames_country.rb
|
75
76
|
- lib/generators/geonames_dump/templates/app/models/geonames_feature.rb
|
77
|
+
- lib/generators/geonames_dump/templates/db/migrate/create_geonames_alternate_names.rb
|
76
78
|
- lib/generators/geonames_dump/templates/db/migrate/create_geonames_countries.rb
|
77
79
|
- lib/generators/geonames_dump/templates/db/migrate/create_geonames_features.rb
|
80
|
+
- lib/generators/geonames_dump/templates/db/migrate/create_geonames_iso_languagecodes.rb
|
78
81
|
- lib/generators/geonames_dump/templates/piktur_config.rb
|
79
82
|
- lib/geonames_dump.rb
|
80
83
|
- lib/geonames_dump/blocks.rb
|