geonames_dump 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +96 -47
- data/lib/generators/geonames_dump/install_generator.rb +4 -4
- data/lib/generators/geonames_dump/templates/app/models/geonames_alternate_name.rb +52 -0
- data/lib/generators/geonames_dump/templates/app/models/geonames_feature.rb +9 -2
- data/lib/generators/geonames_dump/templates/db/migrate/create_geonames_alternate_names.rb +32 -0
- data/lib/generators/geonames_dump/templates/db/migrate/create_geonames_countries.rb +2 -10
- data/lib/generators/geonames_dump/templates/db/migrate/create_geonames_features.rb +20 -25
- data/lib/generators/geonames_dump/templates/db/migrate/create_geonames_iso_languagecodes.rb +17 -0
- data/lib/geonames_dump.rb +31 -0
- data/lib/geonames_dump/version.rb +1 -1
- data/lib/tasks/import.rake +37 -17
- data/lib/tasks/install.rake +1 -1
- data/lib/tasks/truncate.rake +7 -4
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 449ac0186316849284960960df5200fa5830053c
|
4
|
+
data.tar.gz: 647d2b017b3c9cdc5ef44c4fde827e74b3492ebd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: df1f520a053206501c209a3b23a9b1cd9385ac18dbc05d91f3176c533dd355853b733aa066e3f9d1afd6f07da856fc2d5a3455797d5027bfe0fd123e939591ef
|
7
|
+
data.tar.gz: 1e3a7f6ed3f2b63dcc83565a532fdad2bb4cb3e70a56006c87e9097a963cedb61e5e26bbc9491cfc6caa37289f54652460a0990e4dbd3dfafdf340594da34dbe
|
data/README.md
CHANGED
@@ -1,80 +1,129 @@
|
|
1
1
|
# GeonamesDump
|
2
2
|
|
3
|
-
GeonamesDump import geographic data from geonames project into your
|
4
|
-
It's a "gem"
|
5
|
-
|
3
|
+
GeonamesDump import geographic data from geonames project into your
|
4
|
+
application, avoiding to use external service like google maps. It's a "gem"
|
5
|
+
version of the
|
6
|
+
application [brownbeagle/geonames](https://github.com/brownbeagle/geonames).
|
7
|
+
Now you only need to include the dependency into your Gemfile and your project
|
8
|
+
will include geonames.
|
6
9
|
|
7
|
-
You're free to use [geocoder](https://github.com/alexreisner/geocoder) or
|
10
|
+
You're free to use [geocoder](https://github.com/alexreisner/geocoder) or
|
11
|
+
[geokit](https://github.com/imajes/geokit) or any other geocoding solution.
|
8
12
|
|
9
13
|
## Installation
|
10
14
|
|
11
15
|
Add this line to your application's Gemfile:
|
12
16
|
|
13
|
-
|
17
|
+
```
|
18
|
+
gem 'geonames_dump'
|
19
|
+
```
|
14
20
|
|
15
21
|
And then execute:
|
16
22
|
|
17
|
-
|
23
|
+
```
|
24
|
+
bundle install
|
25
|
+
```
|
18
26
|
|
19
27
|
Or install it yourself as:
|
20
28
|
|
21
|
-
|
29
|
+
```
|
30
|
+
gem install geonames_dump
|
31
|
+
```
|
22
32
|
|
23
33
|
## Usage
|
24
34
|
|
25
35
|
Create models and migration files
|
26
36
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
37
|
+
```
|
38
|
+
rails generate geonames_dump:install
|
39
|
+
```
|
40
|
+
|
41
|
+
Import data (takes a loonnnng time!), it will download data, import countries
|
42
|
+
and many features (Countries, Cities having more than 15000 people, Admin1
|
43
|
+
(first administrative subdivision), Admin2 (second level administrative
|
44
|
+
subdivision))
|
45
|
+
|
46
|
+
```
|
47
|
+
rake geonames_dump:install
|
48
|
+
```
|
49
|
+
|
50
|
+
If you need more fine grained control over the installation process you can run
|
51
|
+
individual geoname rake tasks instead of the all-in-one install :
|
52
|
+
|
53
|
+
```
|
54
|
+
$ rake -T | grep geonames_dump
|
55
|
+
|
56
|
+
rake geonames_dump:import:all # Import ALL geonames data.
|
57
|
+
rake geonames_dump:import:many # Import most of geonames data.
|
58
|
+
|
59
|
+
rake geonames_dump:import:admin1 # Import admin1 codes
|
60
|
+
rake geonames_dump:import:admin2 # Import admin2 codes
|
61
|
+
rake geonames_dump:import:cities # Import all cities, regardless of population.
|
62
|
+
rake geonames_dump:import:cities1000 # Import cities with population greater than 1000
|
63
|
+
rake geonames_dump:import:cities15000 # Import cities with population greater than 15000
|
64
|
+
rake geonames_dump:import:cities5000 # Import cities with population greater than 5000
|
65
|
+
rake geonames_dump:import:countries # Import countries informations
|
66
|
+
rake geonames_dump:import:features # Import feature data.
|
67
|
+
rake geonames_dump:import:alternate_names # Import alternate names
|
68
|
+
|
69
|
+
rake geonames_dump:truncate:all # Truncate all geonames data.
|
70
|
+
rake geonames_dump:truncate:countries # Truncate countries informations
|
71
|
+
rake geonames_dump:truncate:admin1 # Truncate admin1 codes
|
72
|
+
rake geonames_dump:truncate:admin2 # Truncate admin2 codes
|
73
|
+
rake geonames_dump:truncate:cities # Truncate cities informations
|
74
|
+
rake geonames_dump:truncate:features # Truncate features informations
|
75
|
+
rake geonames_dump:truncate:alternate_names # Import alternate names
|
76
|
+
```
|
55
77
|
|
56
78
|
## Geonames data usage
|
57
79
|
|
58
|
-
The above commands will import geonames data in your Rails application, in
|
80
|
+
The above commands will import geonames data in your Rails application, in
|
81
|
+
other words, this will create models and fill database with place/city/country
|
82
|
+
informations.
|
83
|
+
|
84
|
+
A convenient way to search for data is to use GeonamesDump search accessor
|
85
|
+
`GeonamesDump.search`. This method interate on data types to find a result.
|
86
|
+
Search order is the following :
|
87
|
+
|
88
|
+
1. Cities
|
89
|
+
2. Alternate names (names in non-latin alphabets)
|
90
|
+
3. First level admin subdivisions
|
91
|
+
4. Second level admin subdivisions
|
92
|
+
5. Countries
|
93
|
+
6. Features (lakes, montains and others various features)
|
94
|
+
|
59
95
|
Now to find a city for example :
|
60
96
|
|
61
|
-
|
97
|
+
```
|
98
|
+
GeonamesDump.search('paris')
|
99
|
+
```
|
62
100
|
|
63
|
-
If your request is ambiguous, like not searching Dublin in Ireland but Dublin
|
101
|
+
If your request is ambiguous, like not searching Dublin in Ireland but Dublin
|
102
|
+
in the USA, you may specify country :
|
64
103
|
|
65
|
-
|
104
|
+
```
|
105
|
+
GeonamesDump.search('dublin').first.country_code
|
106
|
+
=> 'IE'
|
107
|
+
GeonamesDump.search('dublin, us').first.country_code
|
108
|
+
=> 'US'
|
109
|
+
```
|
66
110
|
|
67
|
-
|
111
|
+
If needed, requested type may be specified too :
|
68
112
|
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
113
|
+
```
|
114
|
+
GeonamesDump.search('dublin', type: :city)
|
115
|
+
GeonamesDump.search('dublin, us', type: :city)
|
116
|
+
GeonamesDump.search('paris', type: :feature)
|
117
|
+
```
|
74
118
|
|
75
|
-
|
119
|
+
The following types are available :
|
76
120
|
|
77
|
-
|
121
|
+
- admin1, for first level of adminstrative subdivision
|
122
|
+
- admin2, for second level of adminstrative subdivision
|
123
|
+
- city, for city names
|
124
|
+
- feature, for generic names including all the above
|
125
|
+
- alternate_name, for names in non-latin alphabets
|
126
|
+
- country, for country names
|
78
127
|
|
79
128
|
## Contributing
|
80
129
|
|
@@ -9,15 +9,15 @@ Description:
|
|
9
9
|
Create Geonames initial migrations files and Geonames model files in your application
|
10
10
|
DESC
|
11
11
|
def copy_migrations_files
|
12
|
-
|
13
|
-
|
12
|
+
Dir.glob(File.join(File.expand_path(File.join('..', 'templates', 'db', 'migrate'), __FILE__), '*')).each do |full_path|
|
13
|
+
file = File.basename(full_path, File.extname(full_path))
|
14
14
|
migration_template File.join('db', 'migrate', "#{file}.rb")
|
15
15
|
end
|
16
16
|
end
|
17
17
|
|
18
18
|
def copy_models_files
|
19
|
-
|
20
|
-
|
19
|
+
Dir.glob(File.join(File.expand_path(File.join('..', 'templates', 'app', 'models'), __FILE__), '*')).each do |full_path|
|
20
|
+
file = File.basename(full_path, File.extname(full_path))
|
21
21
|
copy_file File.join('app', 'models', "#{file}.rb")
|
22
22
|
end
|
23
23
|
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
class GeonamesAlternateName < ActiveRecord::Base
|
2
|
+
validates_uniqueness_of :alternate_name_id
|
3
|
+
validates_uniqueness_of :geonameid
|
4
|
+
before_save :set_alternate_name_first_letters
|
5
|
+
|
6
|
+
##
|
7
|
+
# default search (by alternate name)
|
8
|
+
#
|
9
|
+
scope :search, lambda { |q|
|
10
|
+
by_alternate_name(q)
|
11
|
+
}
|
12
|
+
|
13
|
+
##
|
14
|
+
# search by isolanguage code
|
15
|
+
#
|
16
|
+
scope :by_isolanguage, lambda { |q|
|
17
|
+
where("isolanguage = ?", q)
|
18
|
+
}
|
19
|
+
|
20
|
+
##
|
21
|
+
# search prefered names
|
22
|
+
#
|
23
|
+
scope :prefered, lambda {
|
24
|
+
where(is_preferred_name: true)
|
25
|
+
}
|
26
|
+
|
27
|
+
##
|
28
|
+
# search by name
|
29
|
+
#
|
30
|
+
scope :by_alternate_name, lambda { |q|
|
31
|
+
ret = self.scoped
|
32
|
+
ret = ret.where("alternate_name_first_letters = ?", q[0...3].downcase)
|
33
|
+
ret = ret.where("alternate_name LIKE ?", "#{q}%")
|
34
|
+
}
|
35
|
+
|
36
|
+
##
|
37
|
+
# Get associated feature
|
38
|
+
#
|
39
|
+
def feature
|
40
|
+
GeonamesFeature.where(geonameid: self.geonameid)
|
41
|
+
end
|
42
|
+
|
43
|
+
protected
|
44
|
+
|
45
|
+
##
|
46
|
+
# Set first letters of name into index column
|
47
|
+
#
|
48
|
+
def set_alternate_name_first_letters
|
49
|
+
self.alternate_name_first_letters = self.alternate_name[0...3].downcase unless self.alternate_name.nil?
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
@@ -5,7 +5,7 @@ class GeonamesFeature < ActiveRecord::Base
|
|
5
5
|
##
|
6
6
|
# Search for feature, searches might include country (separated by ',')
|
7
7
|
#
|
8
|
-
scope :search, lambda { |query|
|
8
|
+
scope :search, lambda { |query|
|
9
9
|
virgule = query.include? ','
|
10
10
|
|
11
11
|
splited = virgule ? query.split(',') : [query]
|
@@ -19,7 +19,7 @@ class GeonamesFeature < ActiveRecord::Base
|
|
19
19
|
|
20
20
|
unless country.nil?
|
21
21
|
geonames_country = GeonamesCountry.search(country).first
|
22
|
-
ret = ret.where(:
|
22
|
+
ret = ret.where(:country_code => geonames_country.iso) unless geonames_country.nil?
|
23
23
|
end
|
24
24
|
|
25
25
|
ret
|
@@ -39,6 +39,13 @@ class GeonamesFeature < ActiveRecord::Base
|
|
39
39
|
ret
|
40
40
|
}
|
41
41
|
|
42
|
+
def alternate_names
|
43
|
+
GeonamesAlternateName.where(geonameid: self.geonameid)
|
44
|
+
end
|
45
|
+
|
46
|
+
|
47
|
+
protected
|
48
|
+
|
42
49
|
##
|
43
50
|
# Set first letters of name into index column
|
44
51
|
#
|
@@ -0,0 +1,32 @@
|
|
1
|
+
class CreateGeonamesAlternateNames < ActiveRecord::Migration
|
2
|
+
# http://download.geonames.org/export/dump/countryInfo.txt
|
3
|
+
# alternateNameId : the id of this alternate name, int
|
4
|
+
# geonameid : geonameId referring to id in table 'geoname', int
|
5
|
+
# isolanguage : iso 639 language code 2- or 3-characters; 4-characters 'post' for postal codes and 'iata','icao' and faac for airport codes, fr_1793 for French Revolution names, abbr for abbreviation, link for a website, varchar(7)
|
6
|
+
# alternate name : alternate name or name variant, varchar(200)
|
7
|
+
# isPreferredName : '1', if this alternate name is an official/preferred name
|
8
|
+
# isShortName : '1', if this is a short name like 'California' for 'State of California'
|
9
|
+
# isColloquial : '1', if this alternate name is a colloquial or slang term
|
10
|
+
# isHistoric : '1', if this alternate name is historic and was used in the past
|
11
|
+
def change
|
12
|
+
create_table :geonames_alternate_names do |t|
|
13
|
+
t.integer :alternate_name_id
|
14
|
+
t.integer :geonameid
|
15
|
+
t.string :isolanguage, length: 7
|
16
|
+
t.string :alternate_name, length: 200
|
17
|
+
t.boolean :is_preferred_name
|
18
|
+
t.boolean :is_short_name
|
19
|
+
t.boolean :is_colloquial
|
20
|
+
t.boolean :is_historic
|
21
|
+
|
22
|
+
t.string :alternate_name_first_letters
|
23
|
+
|
24
|
+
t.timestamps
|
25
|
+
end
|
26
|
+
|
27
|
+
add_index :geonames_alternate_names, :alternate_name_id
|
28
|
+
add_index :geonames_alternate_names, :geonameid
|
29
|
+
add_index :geonames_alternate_names, :isolanguage
|
30
|
+
add_index :geonames_alternate_names, :alternate_name_first_letters
|
31
|
+
end
|
32
|
+
end
|
@@ -1,9 +1,6 @@
|
|
1
|
-
|
2
|
-
# http://download.geonames.org/export/dump/countryInfo.txt
|
3
|
-
|
4
1
|
class CreateGeonamesCountries < ActiveRecord::Migration
|
5
|
-
|
6
|
-
def
|
2
|
+
# http://download.geonames.org/export/dump/countryInfo.txt
|
3
|
+
def change
|
7
4
|
create_table :geonames_countries do |t|
|
8
5
|
t.string :iso
|
9
6
|
t.string :iso3
|
@@ -29,10 +26,5 @@ class CreateGeonamesCountries < ActiveRecord::Migration
|
|
29
26
|
end
|
30
27
|
|
31
28
|
add_index :geonames_countries, :geonameid
|
32
|
-
add_index :geonames_countries, :country
|
33
|
-
end
|
34
|
-
|
35
|
-
def self.down
|
36
|
-
drop_table :geonames_countries
|
37
29
|
end
|
38
30
|
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
class CreateGeonamesFeatures < ActiveRecord::Migration
|
2
|
-
|
2
|
+
# http://download.geonames.org/export/dump/readme.txt
|
3
3
|
# geonameid : integer id of record in geonames database
|
4
4
|
# name : name of geographical point (utf8) varchar(200)
|
5
5
|
# asciiname : name of geographical point in plain ascii characters, varchar(200)
|
@@ -11,54 +11,49 @@ class CreateGeonamesFeatures < ActiveRecord::Migration
|
|
11
11
|
# country code : ISO-3166 2-letter country code, 2 characters
|
12
12
|
# cc2 : alternate country codes, comma separated, ISO-3166 2-letter country code, 60 characters
|
13
13
|
# admin1 code : fipscode (subject to change to iso code), isocode for the us and ch, see file admin1Codes.txt for display names of this code; varchar(20)
|
14
|
-
# admin2 code : code for the second administrative division, a county in the US, see file admin2Codes.txt; varchar(80)
|
14
|
+
# admin2 code : code for the second administrative division, a county in the US, see file admin2Codes.txt; varchar(80)
|
15
15
|
# admin3 code : code for third level administrative division, varchar(20)
|
16
16
|
# admin4 code : code for fourth level administrative division, varchar(20)
|
17
|
-
# population : bigint (4 byte int)
|
17
|
+
# population : bigint (4 byte int)
|
18
18
|
# elevation : in meters, integer
|
19
|
-
#
|
19
|
+
# dem : digital elevation model, srtm3 or gtopo30, average elevation of 3''x3'' (ca 90mx90m) or 30''x30'' (ca 900mx900m) area in meters, integer. srtm processed by cgiar/ciat.
|
20
20
|
# timezone : the timezone id (see file timeZone.txt)
|
21
21
|
# modification date : date of last modification in yyyy-MM-dd format
|
22
|
-
def
|
22
|
+
def change
|
23
23
|
create_table :geonames_features do |t|
|
24
24
|
t.integer :geonameid
|
25
|
-
t.string :name
|
26
|
-
t.string :asciiname
|
27
|
-
t.string :alternatenames
|
25
|
+
t.string :name, length: 200
|
26
|
+
t.string :asciiname, length: 200
|
27
|
+
t.string :alternatenames, length: 5000
|
28
28
|
t.float :latitude
|
29
29
|
t.float :longitude
|
30
30
|
t.string :feature_class
|
31
|
-
t.string :
|
32
|
-
t.string :
|
33
|
-
t.string :cc2
|
34
|
-
t.string :
|
35
|
-
t.string :
|
36
|
-
t.string :
|
37
|
-
t.string :
|
31
|
+
t.string :feature_code, length: 10
|
32
|
+
t.string :country_code
|
33
|
+
t.string :cc2, length: 60
|
34
|
+
t.string :admin1_code, length: 20
|
35
|
+
t.string :admin2_code, length: 80
|
36
|
+
t.string :admin3_code, length: 20
|
37
|
+
t.string :admin4_code, length: 20
|
38
38
|
t.integer :population
|
39
39
|
t.integer :elevation
|
40
|
-
t.integer :
|
41
|
-
t.string :timezone
|
40
|
+
t.integer :dem
|
41
|
+
t.string :timezone, length: 40
|
42
42
|
t.timestamp :modification
|
43
|
+
|
43
44
|
t.string :type
|
44
|
-
# OPTIMIZE: create another table for all names (alternate names)
|
45
45
|
t.string :asciiname_first_letters
|
46
46
|
|
47
|
-
|
48
47
|
t.timestamps
|
49
48
|
end
|
50
49
|
|
51
50
|
add_index :geonames_features, :geonameid
|
52
51
|
add_index :geonames_features, :name
|
53
52
|
add_index :geonames_features, :asciiname
|
54
|
-
add_index :geonames_features, :
|
53
|
+
add_index :geonames_features, :country_code
|
55
54
|
add_index :geonames_features, :population
|
56
|
-
add_index :geonames_features, :
|
55
|
+
add_index :geonames_features, :admin1_code
|
57
56
|
add_index :geonames_features, :type
|
58
57
|
add_index :geonames_features, :asciiname_first_letters
|
59
58
|
end
|
60
|
-
|
61
|
-
def self.down
|
62
|
-
drop_table :geonames_features
|
63
|
-
end
|
64
59
|
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
class CreateGeonamesIsoLanguagecodes < ActiveRecord::Migration
|
2
|
+
# http://download.geonames.org/export/dump/iso-languagecodes.txt
|
3
|
+
# iso_639_3 : ISO 639-3 language code
|
4
|
+
# iso_639_2 : ISO 639-2 language code
|
5
|
+
# iso_639_1 : ISO 639-1 language code
|
6
|
+
# language_name : language name
|
7
|
+
def change
|
8
|
+
create_table :geonames_iso_languagecodes do |t|
|
9
|
+
t.string :iso_639_3, length: 4
|
10
|
+
t.string :iso_639_2, length: 4
|
11
|
+
t.string :iso_639_1, length: 4
|
12
|
+
t.string :language_name, length: 80
|
13
|
+
|
14
|
+
t.timestamps
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
data/lib/geonames_dump.rb
CHANGED
@@ -5,4 +5,35 @@ require "geonames_dump/railtie" #if defined?(Rails)
|
|
5
5
|
module GeonamesDump
|
6
6
|
# TODO: remove this logger
|
7
7
|
#ActiveRecord::Base.logger = Logger.new(STDOUT) if Rails.env.development? && !ENV['SILENT']
|
8
|
+
|
9
|
+
def self.search(query, options = {})
|
10
|
+
ret = nil
|
11
|
+
|
12
|
+
type = options[:type] || :auto
|
13
|
+
begin
|
14
|
+
case type
|
15
|
+
when :auto
|
16
|
+
# city name
|
17
|
+
ret = GeonamesCity.search(query)
|
18
|
+
# alternate name
|
19
|
+
ret = GeonamesAlternateName.search(query) if ret.blank?
|
20
|
+
# admin1
|
21
|
+
ret = GeonamesAdmin1.search(query) if ret.blank?
|
22
|
+
# admin2
|
23
|
+
ret = GeonamesAdmin2.search(query) if ret.blank?
|
24
|
+
# country
|
25
|
+
ret = GeonamesCountry.search(query) if ret.blank?
|
26
|
+
# feature
|
27
|
+
ret = GeonamesFeature.search(query) if ret.blank?
|
28
|
+
else
|
29
|
+
model = "geonames_#{type.to_s}".camelcase.constantize
|
30
|
+
ret = model.search(query)
|
31
|
+
end
|
32
|
+
rescue NameError => e
|
33
|
+
Rails.logger.error "Unknown type for GeonamesDump : #{e}"
|
34
|
+
#raise
|
35
|
+
end
|
36
|
+
|
37
|
+
ret
|
38
|
+
end
|
8
39
|
end
|
data/lib/tasks/import.rake
CHANGED
@@ -8,9 +8,14 @@ namespace :geonames_dump do
|
|
8
8
|
CACHE_DIR = "#{Rails.root}/db/geonames_cache"
|
9
9
|
|
10
10
|
GEONAMES_FEATURES_COL_NAME = [
|
11
|
-
:geonameid, :name, :asciiname, :alternatenames, :latitude, :longitude,
|
12
|
-
:
|
13
|
-
:
|
11
|
+
:geonameid, :name, :asciiname, :alternatenames, :latitude, :longitude,
|
12
|
+
:feature_class, :feature_code, :country_code, :cc2, :admin1_code,
|
13
|
+
:admin2_code, :admin3_code, :admin4_code, :population, :elevation,
|
14
|
+
:dem, :timezone, :modification
|
15
|
+
]
|
16
|
+
GEONAMES_ALTERNATE_NAMES_COL_NAME = [
|
17
|
+
:alternate_name_id, :geonameid, :isolanguage, :alternate_name,
|
18
|
+
:is_preferred_name, :is_short_name, :is_colloquial, :is_historic
|
14
19
|
]
|
15
20
|
GEONAMES_COUNTRIES_COL_NAME = [
|
16
21
|
:iso, :iso3, :iso_numeric, :fips, :country, :capital, :area, :population, :continent,
|
@@ -30,7 +35,7 @@ namespace :geonames_dump do
|
|
30
35
|
task :all => [:many, :features]
|
31
36
|
|
32
37
|
desc 'Import most of geonames data. Recommended after a clean install.'
|
33
|
-
task :many => [:prepare, :countries, :
|
38
|
+
task :many => [:prepare, :countries, :cities15000, :admin1, :admin2]
|
34
39
|
|
35
40
|
desc 'Import all cities, regardless of population.'
|
36
41
|
task :cities => [:prepare, :cities15000, :cities5000, :cities1000]
|
@@ -44,8 +49,6 @@ namespace :geonames_dump do
|
|
44
49
|
|
45
50
|
# Import into the database.
|
46
51
|
File.open(txt_file) do |f|
|
47
|
-
#VALID_FEATURES = %w[ADMIN1]
|
48
|
-
#feature_attrs = VALID_FEATURES & ENV.keys
|
49
52
|
# TODO: add feature selection
|
50
53
|
insert_data(f, GEONAMES_FEATURES_COL_NAME, GeonamesCity, :title => "Features")
|
51
54
|
end
|
@@ -58,7 +61,6 @@ namespace :geonames_dump do
|
|
58
61
|
|
59
62
|
txt_file = get_or_download("http://download.geonames.org/export/dump/cities#{population}.zip")
|
60
63
|
|
61
|
-
# Import into the database.
|
62
64
|
File.open(txt_file) do |f|
|
63
65
|
insert_data(f, GEONAMES_FEATURES_COL_NAME, GeonamesCity, :title => "cities of #{population}")
|
64
66
|
end
|
@@ -66,10 +68,29 @@ namespace :geonames_dump do
|
|
66
68
|
end
|
67
69
|
|
68
70
|
desc 'Import countries informations'
|
69
|
-
task :countries => :environment do
|
71
|
+
task :countries => [:prepare, :environment] do
|
70
72
|
txt_file = get_or_download('http://download.geonames.org/export/dump/countryInfo.txt')
|
71
73
|
|
72
|
-
|
74
|
+
File.open(txt_file) do |f|
|
75
|
+
insert_data(f, GEONAMES_COUNTRIES_COL_NAME, GeonamesCountry, :title => "Countries")
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
desc 'Import alternate names'
|
80
|
+
task :alternate_names => [:prepare, :environment] do
|
81
|
+
txt_file = get_or_download('http://download.geonames.org/export/dump/alternateNames.zip',
|
82
|
+
txt_file: 'alternateNames.txt')
|
83
|
+
|
84
|
+
File.open(txt_file) do |f|
|
85
|
+
insert_data(f, GEONAMES_ALTERNATE_NAMES_COL_NAME, GeonamesAlternateName, :title => "Alternate names", :buffer => 10000)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
desc 'Import iso language codes'
|
90
|
+
task :language_codes => [:prepare, :environment] do
|
91
|
+
txt_file = get_or_download('http://download.geonames.org/export/dump/alternateNames.zip',
|
92
|
+
txt_file: 'iso-languagecodes.txt')
|
93
|
+
|
73
94
|
File.open(txt_file) do |f|
|
74
95
|
insert_data(f, GEONAMES_COUNTRIES_COL_NAME, GeonamesCountry, :title => "Countries")
|
75
96
|
end
|
@@ -79,15 +100,14 @@ namespace :geonames_dump do
|
|
79
100
|
task :admin1 => [:prepare, :environment] do
|
80
101
|
txt_file = get_or_download('http://download.geonames.org/export/dump/admin1CodesASCII.txt')
|
81
102
|
|
82
|
-
# Import into the database.
|
83
103
|
File.open(txt_file) do |f|
|
84
104
|
insert_data(f, GEONAMES_ADMINS_COL_NAME, GeonamesAdmin1, :title => "Admin1 subdivisions") do |klass, attributes, col_value, idx|
|
85
105
|
col_value.gsub!('(general)', '')
|
86
106
|
col_value.strip!
|
87
107
|
if idx == 0
|
88
108
|
country, admin1 = col_value.split('.')
|
89
|
-
attributes[:
|
90
|
-
attributes[:
|
109
|
+
attributes[:country_code] = country.strip
|
110
|
+
attributes[:admin1_code] = admin1.strip rescue nil
|
91
111
|
else
|
92
112
|
attributes[GEONAMES_ADMINS_COL_NAME[idx]] = col_value
|
93
113
|
end
|
@@ -99,15 +119,14 @@ namespace :geonames_dump do
|
|
99
119
|
task :admin2 => [:prepare, :environment] do
|
100
120
|
txt_file = get_or_download('http://download.geonames.org/export/dump/admin2Codes.txt')
|
101
121
|
|
102
|
-
# Import into the database.
|
103
122
|
File.open(txt_file) do |f|
|
104
123
|
insert_data(f, GEONAMES_ADMINS_COL_NAME, GeonamesAdmin2, :title => "Admin2 subdivisions") do |klass, attributes, col_value, idx|
|
105
124
|
col_value.gsub!('(general)', '')
|
106
125
|
if idx == 0
|
107
126
|
country, admin1, admin2 = col_value.split('.')
|
108
|
-
attributes[:
|
109
|
-
attributes[:
|
110
|
-
attributes[:
|
127
|
+
attributes[:country_code] = country.strip
|
128
|
+
attributes[:admin1_code] = admin1.strip #rescue nil
|
129
|
+
attributes[:admin2_code] = admin2.strip #rescue nil
|
111
130
|
else
|
112
131
|
attributes[GEONAMES_ADMINS_COL_NAME[idx]] = col_value
|
113
132
|
end
|
@@ -179,6 +198,7 @@ namespace :geonames_dump do
|
|
179
198
|
# Setup nice progress output.
|
180
199
|
file_size = file_fd.stat.size
|
181
200
|
title = options[:title] || 'Feature Import'
|
201
|
+
buffer = options[:buffer] || 1000
|
182
202
|
progress_bar = ProgressBar.create(:title => title, :total => file_size, :format => '%a |%b>%i| %p%% %t')
|
183
203
|
|
184
204
|
# create block array
|
@@ -223,7 +243,7 @@ namespace :geonames_dump do
|
|
223
243
|
end
|
224
244
|
|
225
245
|
# increase import speed by performing insert using transaction
|
226
|
-
if line_counter %
|
246
|
+
if line_counter % buffer == 0
|
227
247
|
ActiveRecord::Base.transaction do
|
228
248
|
blocks.call_and_reset
|
229
249
|
end
|
data/lib/tasks/install.rake
CHANGED
data/lib/tasks/truncate.rake
CHANGED
@@ -23,18 +23,21 @@ namespace :geonames_dump do
|
|
23
23
|
task :cities => :environment do
|
24
24
|
GeonamesCity.delete_all #&& GeonamesCity.reset_pk_sequence
|
25
25
|
end
|
26
|
-
|
26
|
+
|
27
27
|
desc 'Truncate countries informations'
|
28
28
|
task :countries => :environment do
|
29
29
|
GeonamesCountry.delete_all && GeonamesCountry.reset_pk_sequence
|
30
30
|
end
|
31
|
-
|
31
|
+
|
32
32
|
desc 'Truncate features informations'
|
33
33
|
task :features => :environment do
|
34
34
|
GeonamesFeature.delete_all && GeonamesFeature.reset_pk_sequence
|
35
35
|
end
|
36
|
-
|
37
|
-
|
36
|
+
|
37
|
+
desc 'Truncate alternate names'
|
38
|
+
task :alternate_names => :environment do
|
39
|
+
GeonamesAlternateName.delete_all && GeonamesAlternateName.reset_pk_sequence
|
40
|
+
end
|
38
41
|
|
39
42
|
end
|
40
43
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: geonames_dump
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alex Pooley
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-11-
|
12
|
+
date: 2013-11-26 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: ruby-progressbar
|
@@ -70,11 +70,14 @@ files:
|
|
70
70
|
- lib/generators/geonames_dump/install_generator.rb
|
71
71
|
- lib/generators/geonames_dump/templates/app/models/geonames_admin1.rb
|
72
72
|
- lib/generators/geonames_dump/templates/app/models/geonames_admin2.rb
|
73
|
+
- lib/generators/geonames_dump/templates/app/models/geonames_alternate_name.rb
|
73
74
|
- lib/generators/geonames_dump/templates/app/models/geonames_city.rb
|
74
75
|
- lib/generators/geonames_dump/templates/app/models/geonames_country.rb
|
75
76
|
- lib/generators/geonames_dump/templates/app/models/geonames_feature.rb
|
77
|
+
- lib/generators/geonames_dump/templates/db/migrate/create_geonames_alternate_names.rb
|
76
78
|
- lib/generators/geonames_dump/templates/db/migrate/create_geonames_countries.rb
|
77
79
|
- lib/generators/geonames_dump/templates/db/migrate/create_geonames_features.rb
|
80
|
+
- lib/generators/geonames_dump/templates/db/migrate/create_geonames_iso_languagecodes.rb
|
78
81
|
- lib/generators/geonames_dump/templates/piktur_config.rb
|
79
82
|
- lib/geonames_dump.rb
|
80
83
|
- lib/geonames_dump/blocks.rb
|