geonames_local 3.3.1 → 3.3.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +13 -10
- data/Guardfile +2 -2
- data/Rakefile +16 -17
- data/geonames.yml +2 -2
- data/geonames_local.gemspec +17 -17
- data/lib/geonames_local/cli.rb +55 -71
- data/lib/geonames_local/data/dump.rb +18 -29
- data/lib/geonames_local/data/export.rb +4 -5
- data/lib/geonames_local/data/shp.rb +7 -8
- data/lib/geonames_local/features/road.rb +7 -10
- data/lib/geonames_local/features/spot.rb +4 -8
- data/lib/geonames_local/features/zone.rb +4 -5
- data/lib/geonames_local/geonames.rb +3 -7
- data/lib/geonames_local/models/mongodb.rb +28 -29
- data/lib/geonames_local/models/postgis.rb +37 -39
- data/lib/geonames_local/version.rb +1 -1
- data/lib/geonames_local.rb +5 -0
- data/spec/geonames_local/cli_spec.rb +2 -2
- data/spec/geonames_local/data/shp_spec.rb +8 -9
- data/spec/geonames_local/features/road_spec.rb +23 -26
- data/spec/geonames_local/features/spot_spec.rb +130 -133
- data/spec/geonames_local/features/zone_spec.rb +21 -22
- data/spec/geonames_local/models/mongodb_spec.rb +1 -1
- data/spec/geonames_local/models/tokyo_spec.rb +21 -21
- data/spec/geonames_local_spec.rb +2 -3
- data/spec/spec_ar_helper.rb +0 -3
- data/spec/spec_helper.rb +2 -4
- data/task/benchmark.rb +6 -8
- data/task/benchmark_cabinet.rb +5 -7
- metadata +9 -8
@@ -1,14 +1,10 @@
|
|
1
|
-
require 'logger'
|
2
|
-
require 'yaml'
|
3
|
-
|
4
1
|
module Geonames
|
5
2
|
Opt = {}
|
6
|
-
Cache = {:
|
3
|
+
Cache = { dump: [], zip: [], roads: [], zones: [] }
|
7
4
|
Codes = YAML.load(File.read(File.join(File.dirname(__FILE__), 'config', 'codes.yml')))
|
8
5
|
|
9
6
|
def info(txt)
|
10
|
-
|
11
|
-
|
12
|
-
end
|
7
|
+
return unless Opt[:verbose]
|
8
|
+
puts(txt) # Logger.info...
|
13
9
|
end
|
14
10
|
end
|
@@ -1,9 +1,13 @@
|
|
1
|
-
require 'mongoid'
|
2
1
|
require 'mongoid_geospatial'
|
3
|
-
require 'geopolitical
|
2
|
+
require 'geopolitical'
|
3
|
+
require 'geopolitical/../../app/models/concerns/geopolitocracy'
|
4
|
+
require 'geopolitical/../../app/models/nation'
|
5
|
+
require 'geopolitical/../../app/models/region'
|
6
|
+
require 'geopolitical/../../app/models/city'
|
7
|
+
require 'geopolitical/../../app/models/hood'
|
4
8
|
|
5
9
|
Mongoid.configure do |config|
|
6
|
-
#config.master = Mongo::Connection.new.db("symbolize_test")
|
10
|
+
# config.master = Mongo::Connection.new.db("symbolize_test")
|
7
11
|
info "Using Mongoid v#{Mongoid::VERSION}"
|
8
12
|
info "Mongoid connecting to #{Opt[:db]}"
|
9
13
|
config.connect_to(Opt[:db][:name])
|
@@ -11,46 +15,45 @@ end
|
|
11
15
|
|
12
16
|
module Geonames
|
13
17
|
module Models
|
14
|
-
require 'geopolitical/../../app/models/hood'
|
15
|
-
require 'geopolitical/../../app/models/city'
|
16
|
-
require 'geopolitical/../../app/models/region'
|
17
|
-
require 'geopolitical/../../app/models/nation'
|
18
18
|
module MongoWrapper
|
19
|
-
|
20
19
|
class << self
|
21
|
-
|
22
|
-
def batch data, clean = false
|
23
|
-
[Region, City].each(&:delete_all) if clean
|
24
|
-
|
20
|
+
def batch(data)
|
25
21
|
@regions, @cities = data[:region], data[:city]
|
26
22
|
@regions.each { |r| create Region, parse_region(r) }
|
27
23
|
@cities.each { |c| create City, parse_city(c) }
|
28
24
|
end
|
29
25
|
|
30
|
-
def
|
26
|
+
def clean
|
27
|
+
[Nation, Region, City].each(&:delete_all)
|
28
|
+
end
|
29
|
+
|
30
|
+
def create(klass, data)
|
31
31
|
# info "#{klass}.new #{data}"
|
32
32
|
klass.create! data
|
33
33
|
rescue => e
|
34
34
|
warn "Prob com spot #{e} #{e.backtrace.join("\n")}"
|
35
35
|
end
|
36
36
|
|
37
|
-
def translate
|
37
|
+
def translate(txt)
|
38
38
|
name_i18n = Opt[:locales].reduce({}) do |h, l|
|
39
|
-
h.merge(
|
39
|
+
h.merge(l => txt)
|
40
40
|
end
|
41
41
|
end
|
42
42
|
|
43
43
|
#
|
44
44
|
# Parse Nations
|
45
45
|
#
|
46
|
-
def nations
|
47
|
-
Nation.delete_all if clean
|
46
|
+
def nations(data)
|
48
47
|
data.each do |row|
|
49
48
|
create Nation, parse_nation(row) rescue nil
|
50
49
|
end
|
51
50
|
end
|
52
51
|
|
53
|
-
def
|
52
|
+
def nations_populated?
|
53
|
+
Nation.count > 0
|
54
|
+
end
|
55
|
+
|
56
|
+
def parse_nation(row)
|
54
57
|
abbr, iso3, ison, fips, name, capital, area, pop, continent,
|
55
58
|
tld, cur_code, cur_name, phone, pos_code, pos_regex,
|
56
59
|
langs, gid, neighbours = row.split(/\t/)
|
@@ -67,13 +70,13 @@ module Geonames
|
|
67
70
|
#
|
68
71
|
# Parse Regions
|
69
72
|
#
|
70
|
-
def parse_region
|
71
|
-
nation = Nation.find_by(abbr: /#{
|
72
|
-
info "Region: #{
|
73
|
+
def parse_region(r)
|
74
|
+
nation = Nation.find_by(abbr: /#{r.nation}/i)
|
75
|
+
info "Region: #{r.name} / #{r.abbr}"
|
73
76
|
{
|
74
|
-
name_translations: translate(
|
75
|
-
gid:
|
76
|
-
nation: nation, code:
|
77
|
+
name_translations: translate(r.name),
|
78
|
+
gid: r.gid, abbr: r.abbr,
|
79
|
+
nation: nation, code: r.region
|
77
80
|
}
|
78
81
|
end
|
79
82
|
|
@@ -97,17 +100,14 @@ module Geonames
|
|
97
100
|
name_translations: translate(s.name),
|
98
101
|
slug: attempt, gid: s.gid, code: s.code,
|
99
102
|
souls: s.pop, geom: [s.lon, s.lat],
|
100
|
-
region: region,
|
103
|
+
region: region, zip: s.zip # tz
|
101
104
|
}
|
102
105
|
end
|
103
|
-
|
104
106
|
end
|
105
|
-
|
106
107
|
end
|
107
108
|
|
108
109
|
# class Nation < Geonames::Spot
|
109
110
|
|
110
|
-
|
111
111
|
# def parse row
|
112
112
|
# end
|
113
113
|
|
@@ -132,6 +132,5 @@ module Geonames
|
|
132
132
|
# belongs_to :city
|
133
133
|
|
134
134
|
# end
|
135
|
-
|
136
135
|
end
|
137
136
|
end
|
@@ -1,61 +1,59 @@
|
|
1
1
|
module Geonames
|
2
2
|
module Models
|
3
|
-
#module Postgis
|
3
|
+
# module Postgis
|
4
4
|
|
5
|
-
|
6
|
-
|
5
|
+
class City < ActiveRecord::Base
|
6
|
+
attr_accessor :x, :y, :z
|
7
7
|
|
8
|
-
|
9
|
-
|
8
|
+
belongs_to :region
|
9
|
+
belongs_to :nation
|
10
10
|
|
11
|
-
|
12
|
-
|
13
|
-
|
11
|
+
validates_presence_of :nation
|
12
|
+
validates_presence_of :name
|
13
|
+
# validates_uniqueness_of :name, :scope => :region_id
|
14
14
|
|
15
|
-
|
16
|
-
|
17
|
-
|
15
|
+
def abbr
|
16
|
+
region.try(:abbr) || nation.abbr
|
17
|
+
end
|
18
18
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
end
|
19
|
+
def geom=(val)
|
20
|
+
self[:geom] = case val
|
21
|
+
when Array then Point.xy(*val)
|
22
|
+
else val
|
24
23
|
end
|
24
|
+
end
|
25
25
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
end
|
26
|
+
# Instantiate self.geom as a Point
|
27
|
+
def validation
|
28
|
+
self.nation ||= region.nation
|
29
|
+
unless !@x || !@y || @x == '' || @y == ''
|
30
|
+
self.geom = Point.from_x_y(@x.to_f, @y.to_f)
|
32
31
|
end
|
33
32
|
end
|
33
|
+
end
|
34
34
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
validates_uniqueness_of :name, :abbr, :scope => :nation_id
|
40
|
-
end
|
35
|
+
class Region < ActiveRecord::Base
|
36
|
+
has_many :cities
|
37
|
+
belongs_to :nation
|
41
38
|
|
42
|
-
|
43
|
-
|
44
|
-
has_many :cities
|
45
|
-
validates_presence_of :name, :abbr
|
46
|
-
validates_uniqueness_of :name, :abbr
|
47
|
-
end
|
39
|
+
validates_uniqueness_of :name, :abbr, scope: :nation_id
|
40
|
+
end
|
48
41
|
|
49
|
-
|
50
|
-
|
51
|
-
|
42
|
+
class Nation < ActiveRecord::Base
|
43
|
+
has_many :regions
|
44
|
+
has_many :cities
|
45
|
+
validates_presence_of :name, :abbr
|
46
|
+
validates_uniqueness_of :name, :abbr
|
47
|
+
end
|
52
48
|
|
49
|
+
class Spot < ActiveRecord::Base
|
50
|
+
validates_presence_of :name
|
51
|
+
end
|
53
52
|
|
54
|
-
#end
|
53
|
+
# end
|
55
54
|
end
|
56
55
|
end
|
57
56
|
|
58
|
-
|
59
57
|
# === Migration
|
60
58
|
|
61
59
|
# Default PG migration:
|
data/lib/geonames_local.rb
CHANGED
@@ -2,20 +2,19 @@ require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
|
|
2
2
|
|
3
3
|
describe SHP do
|
4
4
|
|
5
|
-
Cache[:roads] = [Road.new([:name
|
6
|
-
Road.new([:name
|
7
|
-
Road.new([:name
|
8
|
-
Road.new([:name
|
5
|
+
Cache[:roads] = [Road.new([:name, :zone, :geom], "R DOUTOR VITAL BRASIL\t\\N\t0105000020E6100000010000000102000020E6100000020000009561DC0DA22B47C0EAB12D03CEF237C0136058FE7C2B47C0DE54A4C2D8F237C0"),
|
6
|
+
Road.new([:name, :zone, :geom], "R DOUTOR VITAL BRASIL\t\\N\t0105000020E6100000010000000102000020E610000003000000136058FE7C2B47C0DE54A4C2D8F237C0094E7D20792B47C0CCB56801DAF237C0CDCAF6216F2B47C08F1B7E37DDF237C0"),
|
7
|
+
Road.new([:name, :zone, :geom], "R DOUTOR VITAL BRASIL\t\\N\t0105000020E6100000010000000102000020E610000003000000A0F99CBB5D2B47C07A8F334DD8F237C019C6DD205A2B47C008E8BE9CD9F237C009C38025572B47C00DA5F622DAF237C0"),
|
8
|
+
Road.new([:name, :zone, :geom], "R DOUTOR VITAL BRASIL\t\\N\t0105000020E6100000010000000102000020E61000000300000009C38025572B47C00DA5F622DAF237C0155454FD4A2B47C082397AFCDEF237C0FB213658382B47C053060E68E9F237C0")]
|
9
9
|
|
10
|
-
Opt[:type] =
|
10
|
+
Opt[:type] = 'road'
|
11
11
|
|
12
|
-
it
|
12
|
+
it 'should merge two records linestrings' do
|
13
13
|
pending
|
14
14
|
@s = SHP.new(nil)
|
15
15
|
r = @s.reduce!
|
16
|
-
r.length.
|
17
|
-
r[0].geom.as_hex_ewkb.
|
16
|
+
expect(r.length).to eql(1)
|
17
|
+
expect(r[0].geom.as_hex_ewkb).not_to eql(Cache[:roads][0].geom.as_hex_ewkb)
|
18
18
|
end
|
19
19
|
|
20
|
-
|
21
20
|
end
|
@@ -1,68 +1,65 @@
|
|
1
|
-
require
|
2
|
-
require "geo_ruby"
|
1
|
+
require 'spec_helper'
|
3
2
|
|
4
3
|
describe Road do
|
5
4
|
|
6
|
-
describe
|
5
|
+
describe 'Parsing Dump' do
|
7
6
|
|
8
7
|
before do
|
9
8
|
@road = Road.new([:zone, :name, :geom], "RIO COMPRIDO\tAV PAULO DE FRONTIN\t0105000020E6100000010000000102000020E61000000700000029A117E0C79A45C0D50B2027B3EE36C0782289C2C99A45C0ED188FFFC5EE36C0E6D342ABCA9A45C0AB8A7A7BD0EE36C00216B5BFCC9A45C07359511DDFEE36C0C489F290CE9A45C0DEA42096F7EE36C0453B2C65CD9A45C0DFCF664816EF36C04089595CCD9A45C00A220DA31AEF36C0")
|
10
9
|
end
|
11
10
|
|
12
|
-
it
|
13
|
-
@road.zone.
|
11
|
+
it 'should parse zone' do
|
12
|
+
expect(@road.zone).to eql('RIO COMPRIDO')
|
14
13
|
end
|
15
14
|
|
16
|
-
it
|
17
|
-
@road.name.
|
15
|
+
it 'should parse code' do
|
16
|
+
expect(@road.name).to eql('AV PAULO DE FRONTIN')
|
18
17
|
end
|
19
18
|
|
20
|
-
it
|
21
|
-
@road.kind.
|
19
|
+
it 'should parse kind' do
|
20
|
+
expect(@road.kind).to eql(:avenue)
|
22
21
|
end
|
23
22
|
|
24
|
-
it
|
25
|
-
@road.geom.
|
23
|
+
it 'should parse geom' do
|
24
|
+
expect(@road.geom).to be_kind_of(GeoRuby::SimpleFeatures::MultiLineString)
|
26
25
|
end
|
27
26
|
|
28
|
-
it
|
29
|
-
@road.geom.
|
27
|
+
it 'should have 1 geometry' do
|
28
|
+
expect(@road.geom.size).to eq(1)
|
30
29
|
end
|
31
30
|
|
32
|
-
it
|
31
|
+
it 'should have 1 geometry linestring' do
|
33
32
|
geom = @road.geom.geometries[0]
|
34
|
-
geom.
|
33
|
+
expect(geom).to be_kind_of(GeoRuby::SimpleFeatures::LineString)
|
35
34
|
end
|
36
35
|
|
37
|
-
it
|
38
|
-
@road.geom.geometries[0].
|
36
|
+
it 'should have 1 geometry linestring with n points' do
|
37
|
+
expect(@road.geom.geometries[0].size).to eq(7)
|
39
38
|
end
|
40
39
|
|
41
40
|
end
|
42
41
|
|
43
|
-
describe
|
42
|
+
describe 'Another parse' do
|
44
43
|
before do
|
45
44
|
@road = Road.new([:length, :name, :null, :null, :geom], "35.487\tMS-380\t2\t\N\t0105000020E6100000010000000102000020E6100000120000005A4B85AD49DC4BC0D9438EB3AE8736C04D5707662DDA4BC07FD132464F8336C05AF6FA9F8FD84BC0C43ACFACA97E36C07DAC68ABFED14BC0ADFF26E9687936C036ED20214BD04BC035B173E5557A36C0C115051259CD4BC0E4C9CF15617636C0429724EA1AC94BC04C0D087A5E7536C04C0FAB81C5C54BC0FD07B789257336C07FB8E40122BE4BC026F0474B218036C053A51A37ABBD4BC0E03173F8698036C0D642B66B34BD4BC0BC4E66A4B28036C0D696B79FBDBC4BC05337214FFB8036C01CA71ED346BC4BC043DCA3F8438136C07879EB05D0BB4BC0212EEEA08C8136C0B5131E3859BB4BC08F1D0048D58136C0A17BB669E2BA4BC0259BD9ED1D8236C007B7B49A6BBA4BC07D977A92668236C0BE2439F759B94BC0B67D375CCC8236C0")
|
46
45
|
end
|
47
46
|
|
48
|
-
it
|
49
|
-
@road.name.
|
47
|
+
it 'should parse code' do
|
48
|
+
expect(@road.name).to eql('MS-380')
|
50
49
|
end
|
51
50
|
|
52
|
-
it
|
53
|
-
@road.kind.
|
51
|
+
it 'should parse kind' do
|
52
|
+
expect(@road.kind).to eql(:road)
|
54
53
|
end
|
55
54
|
|
56
|
-
it
|
57
|
-
@road.geom.
|
55
|
+
it 'should parse geom' do
|
56
|
+
expect(@road.geom).to be_kind_of(GeoRuby::SimpleFeatures::MultiLineString)
|
58
57
|
end
|
59
58
|
|
60
59
|
end
|
61
60
|
end
|
62
61
|
|
63
|
-
|
64
62
|
# RIO COMPRIDO\tAV PAULO DE FRONTIN\t0105000020E6100000010000000102000020E61000000700000029A117E0C79A45C0D50B2027B3EE36C0782289C2C99A45C0ED188FFFC5EE36C0E6D342ABCA9A45C0AB8A7A7BD0EE36C00216B5BFCC9A45C07359511DDFEE36C0C489F290CE9A45C0DEA42096F7EE36C0453B2C65CD9A45C0DFCF664816EF36C04089595CCD9A45C00A220DA31AEF36C0
|
65
63
|
# RIO COMPRIDO\tELEV ENG FREYSSINET\t0105000020E6100000010000000102000020E61000001600000029A117E0C79A45C0D50B2027B3EE36C0616EE6DEC39A45C03A36CF0E94EE36C0E8C4CE25C09A45C0C7CC5FA884EE36C06FC623BBAD9A45C0CC16549751EE36C0C886FA2AA99A45C03FFE37E23EEE36C0D8A69F2DA79A45C036DFE40921EE36C0C5B0C586A89A45C060E6BA0F0FEE36C0417340F4B29A45C0CCD0A07BB9ED36C0F28BB536B89A45C01F0BB3CE94ED36C038AED593BA9A45C0AD84B8EA77ED36C0C8D80C30BD9A45C0DB5E15A837ED36C0642CEC83C29A45C04E78D9AEA9EC36C01FF44297C79A45C08091F28337EC36C0D0C63FCECA9A45C0ACC190D2E7EB36C06160C265CC9A45C0F4BA5997ABEB36C0141C44A2D19A45C046DA3ABC0BEB36C00A5183F3D49A45C0A800B9F987EA36C08F84CC8CD69A45C078B5508D41EA36C0FDEFEEEDD49A45C06F668B46D8E936C06DF2C8EBD39A45C05C575FADBEE936C05718F576D39A45C0D31DF017B1E936C056E59516D29A45C0F9EE1635A7E936C0
|
66
64
|
# GAMBOA\tTUN JOAO RICARDO\t0105000020E6100000010000000102000020E6100000060000008156E6D6F19845C06D89F7CECCE536C00A1B72A6CF9845C07AAC746217E636C016E1A9D1C99845C0A3B9AA0C24E636C0EE90F8C5C49845C047B01D022FE636C05F1D9379BB9845C0ADEA1C3443E636C0B146DDF6B19845C05FC1548D57E636C0
|
67
65
|
# SANTO CRISTO\tVD S PEDRO E S PAULO\t0105000020E6100000010000000102000020E610000006000000AC45D194B79945C04409BB661DE636C0B49FF422B89945C081E6846525E636C022FF9BAC949945C031F0CFDF8FE636C0978E80ED8F9945C056113F209EE636C0AF1242F7879945C001FE5FA5B8E636C09AB3B4607D9945C0A38B9395DCE636C0
|
68
|
-
|