geonames_local 3.1.3 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/{README.rdoc → README.md} +11 -25
- data/geonames_local.gemspec +8 -7
- data/lib/geonames_local/cli.rb +105 -100
- data/lib/geonames_local/config/geonames.yml +1 -1
- data/lib/geonames_local/models/mongodb.rb +3 -1
- data/lib/geonames_local/version.rb +1 -1
- metadata +36 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fdef4304a7290c1a586d7e9b437153021bbc6b37
|
4
|
+
data.tar.gz: 41eb2353c1e671d5e6986922221aed29563eb5d5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 01b20afbb5e96032c2882dde15dbc7025e77bd39a50884b6f54f4bf8acc4981273e715eeb472f39cf1db1f4d0c437f8dad4c07640d4506f10042d9024c643ed7
|
7
|
+
data.tar.gz: 5f194b2ddc720aa9fdaeb6ef3fd155c52933dba1bd3b9abda4a3c48fca37af590e398144a0e515f13c69112656468f60855b397f64f87bc663fd3cb34e51d98f
|
data/{README.rdoc → README.md}
RENAMED
@@ -1,11 +1,13 @@
|
|
1
|
-
|
1
|
+
Geonames Local
|
2
|
+
==============
|
2
3
|
|
3
4
|
Downloads and store Geonames.org data locally (MongoDB, Mongoid).
|
4
5
|
Making every Geoname API operation possible on your servers.
|
5
6
|
No hit limit, fast as possible.
|
6
7
|
|
7
8
|
|
8
|
-
|
9
|
+
Use
|
10
|
+
---
|
9
11
|
|
10
12
|
Install where you want to populate the DB:
|
11
13
|
|
@@ -18,13 +20,13 @@ You will also need in your system:
|
|
18
20
|
* curl
|
19
21
|
|
20
22
|
|
21
|
-
|
22
|
-
|
23
|
+
Config
|
24
|
+
------
|
23
25
|
|
24
26
|
Create a config yml file:
|
25
27
|
|
26
28
|
|
27
|
-
geonames
|
29
|
+
geonames init
|
28
30
|
|
29
31
|
|
30
32
|
Will generate a "geonames.yml" file on your folder.
|
@@ -43,7 +45,8 @@ To populate the nations database for the first time use:
|
|
43
45
|
geonames -c geoconfig.yml nations
|
44
46
|
|
45
47
|
|
46
|
-
|
48
|
+
Adapters
|
49
|
+
--------
|
47
50
|
|
48
51
|
So, supposing Mongoid, something like this is possible:
|
49
52
|
|
@@ -59,24 +62,7 @@ Be sure to use a database based on the PostGIS template.
|
|
59
62
|
|
60
63
|
|
61
64
|
|
62
|
-
|
65
|
+
Next
|
66
|
+
----
|
63
67
|
|
64
|
-
- Local relational mapping of geonames data
|
65
|
-
- Map geonames fields to your db scheme
|
66
|
-
- Countries are a static yml file :/
|
67
68
|
- IP Geonames? http://ipinfodb.com
|
68
|
-
|
69
|
-
|
70
|
-
== Note on Patches/Pull Requests
|
71
|
-
|
72
|
-
* Fork the project.
|
73
|
-
* Make your feature addition or bug fix.
|
74
|
-
* Add tests for it. This is important so I don't break it in a
|
75
|
-
future version unintentionally.
|
76
|
-
* Commit, do not mess with rakefile, version, or history.
|
77
|
-
(if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
|
78
|
-
* Send me a pull request. Bonus points for topic branches.
|
79
|
-
|
80
|
-
== Copyright
|
81
|
-
|
82
|
-
Copyright (c) 2009 Marcos Piccinini. See LICENSE for details.
|
data/geonames_local.gemspec
CHANGED
@@ -19,21 +19,22 @@ Gem::Specification.new do |gem|
|
|
19
19
|
gem.require_paths = ["lib"]
|
20
20
|
gem.summary = %q{Dumps geonames data for local usage}
|
21
21
|
|
22
|
+
|
22
23
|
gem.extra_rdoc_files = [
|
23
24
|
"MIT-LICENSE",
|
24
|
-
"README.
|
25
|
+
"README.md"
|
25
26
|
]
|
26
27
|
|
28
|
+
gem.add_dependency('mongoid', ['>= 3.0.0'])
|
29
|
+
gem.add_dependency('geopolitical', ['>= 0.8.0'])
|
30
|
+
|
27
31
|
gem.post_install_message = %q{
|
28
32
|
Geonames Local
|
29
33
|
--------------
|
30
34
|
|
31
|
-
|
32
|
-
|
33
|
-
PostgreSQL => pg
|
34
|
-
MongoDB => mongoid (optional: mongo_ext)
|
35
|
-
Tokyo => tokyocabinet
|
35
|
+
Use `geonames init` to create a config.yml file.
|
36
|
+
Or `geonames -c config.yml` to run using a config file.
|
36
37
|
|
37
|
-
Have fun
|
38
|
+
Have fun!
|
38
39
|
}
|
39
40
|
end
|
data/lib/geonames_local/cli.rb
CHANGED
@@ -71,123 +71,128 @@ module Geonames
|
|
71
71
|
exit
|
72
72
|
end
|
73
73
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
load_config
|
74
|
+
# Ugly but works?
|
75
|
+
def work(argv)
|
76
|
+
trap(:INT) { stop! }
|
77
|
+
trap(:TERM) { stop! }
|
78
|
+
Opt.merge! parse_options(argv)
|
79
|
+
if Opt[:locales].nil? || Opt[:locales].empty?
|
80
|
+
Opt[:locales] = ['en']
|
81
|
+
end
|
84
82
|
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
83
|
+
if shp = Opt[:shp]
|
84
|
+
SHP.import(shp)
|
85
|
+
exit
|
86
|
+
end
|
89
87
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
88
|
+
#
|
89
|
+
# Return Codes and Exit
|
90
|
+
#
|
91
|
+
if argv[0] =~ /list|codes/
|
92
|
+
Codes.each do |key,val|
|
93
|
+
str = [val.values, key.to_s].join(" ").downcase
|
94
|
+
if s = argv[1]
|
95
|
+
next unless str =~ /#{s.downcase}/
|
96
|
+
end
|
97
|
+
puts "#{val[:en_us]}: #{key}"
|
98
98
|
end
|
99
|
-
|
99
|
+
exit
|
100
100
|
end
|
101
|
-
exit
|
102
|
-
end
|
103
101
|
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
102
|
+
#
|
103
|
+
# If arguments scaffold, config, write down yml.
|
104
|
+
#
|
105
|
+
if argv[0] =~ /scaff|conf|init/
|
106
|
+
fname = (argv[1] || "geonames") + ".yml"
|
107
|
+
if File.exist?(fname)
|
108
|
+
puts "File exists."
|
109
|
+
else
|
110
|
+
puts "Writing to #{fname}"
|
111
|
+
`cp #{File.join(File.dirname(__FILE__), 'config', 'geonames.yml')} #{fname}`
|
112
|
+
end
|
113
|
+
exit
|
114
114
|
end
|
115
|
-
exit
|
116
|
-
end
|
117
115
|
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
if Opt[:nations].empty? || argv[0] =~ /coun|nati/
|
138
|
-
info "\nPopulating 'nations' database..."
|
139
|
-
dump = Geonames::Dump.new(:all, :dump)
|
140
|
-
info "\n---\nTotal #{dump.data.length} parsed."
|
141
|
-
|
142
|
-
info "Writing to nations DB"
|
143
|
-
Geonames::Models::MongoWrapper.nations dump.data, Opt[:clean]
|
144
|
-
|
145
|
-
# Regions, Cities....
|
116
|
+
#
|
117
|
+
# Require georuby optionally
|
118
|
+
#
|
119
|
+
require "geo_ruby" if Opt[:mapping] && Opt[:mapping][:geom]
|
120
|
+
|
121
|
+
#
|
122
|
+
# Load config if we got til here
|
123
|
+
#
|
124
|
+
load_config
|
125
|
+
|
126
|
+
#
|
127
|
+
# Export Data as CSV or JSON
|
128
|
+
#
|
129
|
+
if argv[0] =~ /csv|json/
|
130
|
+
Geonames::Export.new(Nation.all).to_csv
|
131
|
+
|
132
|
+
#
|
133
|
+
# Do the magic! Import Geonames Data
|
134
|
+
#
|
146
135
|
else
|
147
|
-
|
148
|
-
|
149
|
-
info "\n---\nTotal #{dump.size} parsed. #{zip.size} zips."
|
136
|
+
load_adapter(Opt[:store])
|
137
|
+
info "Using adapter #{Opt[:store]}.."
|
150
138
|
|
151
|
-
|
152
|
-
|
139
|
+
# Nations
|
140
|
+
if Opt[:nations].empty? || argv[0] =~ /coun|nati/
|
141
|
+
info "\nPopulating 'nations' database..."
|
142
|
+
dump = Geonames::Dump.new(:all, :dump)
|
143
|
+
info "\n---\nTotal #{dump.data.length} parsed."
|
153
144
|
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
145
|
+
info "Writing to nations DB"
|
146
|
+
Geonames::Models::MongoWrapper.nations dump.data, Opt[:clean]
|
147
|
+
|
148
|
+
# Regions, Cities....
|
149
|
+
else
|
150
|
+
zip = Geonames::Dump.new(Opt[:nations], :zip).data
|
151
|
+
dump = Geonames::Dump.new(Opt[:nations], :dump).data
|
152
|
+
info "\n---\nTotal #{dump.size} parsed. #{zip.size} zips."
|
153
|
+
|
154
|
+
info "Join dump << zip"
|
155
|
+
dump = unify!(dump, zip).group_by(&:kind)
|
156
|
+
|
157
|
+
info "Writing to DB..."
|
158
|
+
Geonames::Models::MongoWrapper.batch dump, Opt[:clean]
|
159
|
+
# info "Writing cities..."
|
160
|
+
# Geonames::Models::City.from_batch dump[:city]
|
161
|
+
end
|
158
162
|
end
|
159
163
|
end
|
160
|
-
end
|
161
164
|
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
165
|
+
def load_adapter(name)
|
166
|
+
begin
|
167
|
+
require "geonames_local/models/#{name}"
|
168
|
+
# rescue LoadError
|
169
|
+
# puts "Can't find adapter for #{name}"
|
170
|
+
# stop!
|
171
|
+
end
|
168
172
|
end
|
169
|
-
end
|
170
173
|
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
174
|
+
def unify! dump, zip
|
175
|
+
start = Time.now
|
176
|
+
dump.map! do |spot|
|
177
|
+
if other = zip.find { |d| d.code == spot.code }
|
178
|
+
spot.zip = other.zip
|
179
|
+
spot
|
180
|
+
else
|
181
|
+
spot
|
182
|
+
end
|
179
183
|
end
|
184
|
+
info "Done. #{(Time.now-start).to_i}s"
|
185
|
+
dump
|
180
186
|
end
|
181
|
-
info "Done. #{(Time.now-start).to_i}s"
|
182
|
-
dump
|
183
|
-
end
|
184
187
|
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
188
|
+
def stop!
|
189
|
+
puts "Closing Geonames..."
|
190
|
+
exit
|
191
|
+
end
|
192
|
+
|
193
|
+
|
194
|
+
end # class < self
|
190
195
|
|
191
|
-
end
|
196
|
+
end # CLI
|
192
197
|
|
193
|
-
end
|
198
|
+
end # Geonames
|
@@ -4,6 +4,7 @@ require 'geopolitical/helpers'
|
|
4
4
|
|
5
5
|
Mongoid.configure do |config|
|
6
6
|
#config.master = Mongo::Connection.new.db("symbolize_test")
|
7
|
+
info "Mongoid connecting to #{Opt[:db]}"
|
7
8
|
config.connect_to(Opt[:db][:name])
|
8
9
|
end
|
9
10
|
|
@@ -34,9 +35,10 @@ module Geonames
|
|
34
35
|
end
|
35
36
|
|
36
37
|
def create klass, data
|
38
|
+
info "#{klass}.new #{data}"
|
37
39
|
klass.create! data
|
38
40
|
rescue => e
|
39
|
-
|
41
|
+
warn "Prob com spot #{e} #{e.backtrace.join("\n")}"
|
40
42
|
end
|
41
43
|
|
42
44
|
def translate txt
|
metadata
CHANGED
@@ -1,15 +1,43 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: geonames_local
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Marcos Piccinini
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-10-
|
12
|
-
dependencies:
|
11
|
+
date: 2013-10-08 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: mongoid
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 3.0.0
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 3.0.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: geopolitical
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.8.0
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 0.8.0
|
13
41
|
description: Dumps geonames data to feed a local db
|
14
42
|
email: x@nofxx.com
|
15
43
|
executables:
|
@@ -17,13 +45,13 @@ executables:
|
|
17
45
|
extensions: []
|
18
46
|
extra_rdoc_files:
|
19
47
|
- MIT-LICENSE
|
20
|
-
- README.
|
48
|
+
- README.md
|
21
49
|
files:
|
22
50
|
- ".gitignore"
|
23
51
|
- Gemfile
|
24
52
|
- Guardfile
|
25
53
|
- MIT-LICENSE
|
26
|
-
- README.
|
54
|
+
- README.md
|
27
55
|
- Rakefile
|
28
56
|
- bin/geonames
|
29
57
|
- geonames.yml
|
@@ -68,13 +96,10 @@ post_install_message: |2
|
|
68
96
|
Geonames Local
|
69
97
|
--------------
|
70
98
|
|
71
|
-
|
99
|
+
Use `geonames init` to create a config.yml file.
|
100
|
+
Or `geonames -c config.yml` to run using a config file.
|
72
101
|
|
73
|
-
|
74
|
-
MongoDB => mongoid (optional: mongo_ext)
|
75
|
-
Tokyo => tokyocabinet
|
76
|
-
|
77
|
-
Have fun because:
|
102
|
+
Have fun!
|
78
103
|
rdoc_options: []
|
79
104
|
require_paths:
|
80
105
|
- lib
|