gis_scraper 0.1.6.pre → 0.1.7.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/gis_scraper/feature_scraper.rb +4 -15
- data/lib/gis_scraper/layer.rb +26 -33
- data/lib/gis_scraper/version.rb +1 -1
- data/lib/gis_scraper.rb +9 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d189134b4efe0805e92351f7352878d54975c1e7
|
4
|
+
data.tar.gz: 5dca0815dea6a1c2f3e5a142aae7b2642f872864
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: baba12b3211806d01533f0e424e34170efc88fe67c3dcb64c4573f88ad4da0fa0e28c83b9cc5abb51949c5bd2ef91b9c6bf6a5fb26bae409925e031015087a2c
|
7
|
+
data.tar.gz: 595016b6687eb634b8f2bc6a98b0e746061c38ea434653e5305bc31da3b3f4563ab5e8e2cc11b88e118cfec1af988d9e08efc1fc2f9a132a987df23cfa489ccf
|
@@ -1,12 +1,3 @@
|
|
1
|
-
class JSONParser < Mechanize::File
|
2
|
-
attr_reader :json
|
3
|
-
|
4
|
-
def initialize(uri=nil, response=nil, body=nil, code=nil)
|
5
|
-
super(uri, response, body, code)
|
6
|
-
@json = JSON.parse(body)
|
7
|
-
end
|
8
|
-
end
|
9
|
-
|
10
1
|
class FeatureScraper
|
11
2
|
|
12
3
|
attr_reader :name
|
@@ -14,11 +5,9 @@ class FeatureScraper
|
|
14
5
|
def initialize(url)
|
15
6
|
@url = url
|
16
7
|
@agent = Mechanize.new
|
17
|
-
@agent.pluggable_parser['text/plain'] = JSONParser
|
18
|
-
@layer = layer # hash
|
19
|
-
@name = name
|
20
|
-
@pk = pk
|
21
|
-
@max = max # maxRecordCount - usually 1000
|
8
|
+
@agent.pluggable_parser['text/plain'] = GisScraper::JSONParser
|
9
|
+
@layer = layer # hash of json
|
10
|
+
@name, @pk, @max = name, pk, max # maxRecordCount - usually 1000
|
22
11
|
@form = form
|
23
12
|
@loops = loops
|
24
13
|
@threads = GisScraper.config[:threads]
|
@@ -57,7 +46,7 @@ class FeatureScraper
|
|
57
46
|
|
58
47
|
def set_query_params(loop_num = nil)
|
59
48
|
@form.fields[0].value = where_text(loop_num)
|
60
|
-
loop_num ? @form.radiobuttons[4].uncheck : @form.radiobuttons[4].check
|
49
|
+
loop_num ? @form.radiobuttons[4].uncheck : @form.radiobuttons[4].check
|
61
50
|
@form.fields[6].value = '*'
|
62
51
|
@form.field_with(name: 'f').options[1].select # for JSON
|
63
52
|
end
|
data/lib/gis_scraper/layer.rb
CHANGED
@@ -1,27 +1,15 @@
|
|
1
1
|
require 'fileutils'
|
2
|
+
require 'tmpdir'
|
2
3
|
|
3
4
|
class Layer
|
4
5
|
|
5
|
-
class JSONParser < Mechanize::File
|
6
|
-
attr_reader :json
|
7
|
-
|
8
|
-
def initialize(uri=nil, response=nil, body=nil, code=nil)
|
9
|
-
super(uri, response, body, code)
|
10
|
-
@json = JSON.parse(body)
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
6
|
class UnknownLayerType < StandardError; end
|
15
7
|
class NoDatabase < StandardError; end
|
16
8
|
class OgrMissing < StandardError; end
|
17
9
|
|
18
10
|
attr_reader :type
|
19
11
|
|
20
|
-
|
21
|
-
'Feature Layer',
|
22
|
-
'Annotation Layer',
|
23
|
-
'Annotation SubLayer']
|
24
|
-
QUERYABLE = ['Feature Layer', 'Annotation Layer']
|
12
|
+
TYPE = %w(Group\ Layer Feature\ Layer Annotation\ Layer Annotation\ SubLayer)
|
25
13
|
|
26
14
|
CONN = [:host, :port, :dbname, :user, :password] # PG connection options
|
27
15
|
|
@@ -31,22 +19,19 @@ class Layer
|
|
31
19
|
'esriGeometryPolyline' => 'MULTILINESTRING',
|
32
20
|
'esriGeometryPolygon' => 'MULTIPOLYGON'}
|
33
21
|
|
34
|
-
|
22
|
+
MSURL = 'MapServer'
|
35
23
|
OGR2OGR = 'ogr2ogr -f "PostgreSQL" PG:'
|
36
24
|
|
37
25
|
def initialize(url, path = nil)
|
38
26
|
@conn_hash = CONN.zip(CONN.map { |key| GisScraper.config[key] }).to_h
|
39
27
|
@url = url
|
40
28
|
@output_path = output_path(path) || config_path
|
41
|
-
@
|
42
|
-
@id = id
|
29
|
+
@id, @mapserver_url = id, mapserver_url # mapserver url ends '../MapServer'
|
43
30
|
@agent = Mechanize.new
|
44
|
-
@agent.pluggable_parser['text/plain'] = JSONParser
|
31
|
+
@agent.pluggable_parser['text/plain'] = GisScraper::JSONParser
|
45
32
|
validate_url
|
46
33
|
@page_json = page_json
|
47
|
-
@type = type
|
48
|
-
@name = name
|
49
|
-
@sub_layer_ids = sub_layer_ids
|
34
|
+
@type, @name, @sub_layer_ids = type, name, sub_layer_ids
|
50
35
|
end
|
51
36
|
|
52
37
|
def output_json
|
@@ -61,8 +46,8 @@ class Layer
|
|
61
46
|
|
62
47
|
private
|
63
48
|
|
64
|
-
def output(format) # recurses sub-layers
|
65
|
-
|
49
|
+
def output(format) # recurses sub-layers, if any (none for Annotation layers)
|
50
|
+
@type == 'Feature Layer' ? method(format) : do_sub_layers(format)
|
66
51
|
end
|
67
52
|
|
68
53
|
def method(format)
|
@@ -87,7 +72,7 @@ class Layer
|
|
87
72
|
File.expand_path GisScraper.config[:output_path]
|
88
73
|
end
|
89
74
|
|
90
|
-
def
|
75
|
+
def mapserver_url
|
91
76
|
@url.split('/')[0..-2].join('/')
|
92
77
|
end
|
93
78
|
|
@@ -97,7 +82,7 @@ class Layer
|
|
97
82
|
|
98
83
|
def validate_url
|
99
84
|
raise ArgumentError, 'URL must end with layer id' if @id.to_i.to_s != @id
|
100
|
-
raise ArgumentError, 'Bad MapServer URL' if @
|
85
|
+
raise ArgumentError, 'Bad MapServer URL' if @mapserver_url[-9..-1] != MSURL
|
101
86
|
end
|
102
87
|
|
103
88
|
def page_json
|
@@ -113,7 +98,7 @@ class Layer
|
|
113
98
|
end
|
114
99
|
|
115
100
|
def validate_type(type)
|
116
|
-
raise UnknownLayerType, type unless (
|
101
|
+
raise UnknownLayerType, type unless (TYPE.any? { |t| t == type })
|
117
102
|
type
|
118
103
|
end
|
119
104
|
|
@@ -121,18 +106,26 @@ class Layer
|
|
121
106
|
@page_json['subLayers'].map { |hash| hash['id'] } || []
|
122
107
|
end
|
123
108
|
|
124
|
-
def json_data
|
125
|
-
FeatureScraper.new(
|
109
|
+
def json_data
|
110
|
+
FeatureScraper.new("#{@mapserver_url}/#{@id}").json_data
|
126
111
|
end
|
127
112
|
|
128
113
|
def write_json
|
129
|
-
|
114
|
+
IO.write json_path, json_data
|
115
|
+
end
|
116
|
+
|
117
|
+
def json_path
|
118
|
+
"#{@output_path}/#{@name}.json"
|
130
119
|
end
|
131
120
|
|
132
121
|
def write_to_db
|
133
|
-
@output_path =
|
134
|
-
|
135
|
-
|
122
|
+
@output_path = Dir.mktmpdir('gis_scraper') # prefix for identification
|
123
|
+
begin
|
124
|
+
write_json
|
125
|
+
`#{OGR2OGR}"#{conn}" "#{json_path}" -nln #{table} #{srs} -nlt #{geom}`
|
126
|
+
ensure
|
127
|
+
FileUtils.remove_entry @output_path
|
128
|
+
end
|
136
129
|
end
|
137
130
|
|
138
131
|
def geom
|
@@ -165,7 +158,7 @@ class Layer
|
|
165
158
|
end
|
166
159
|
|
167
160
|
def sub_layer(id, path)
|
168
|
-
Layer.new("#{@
|
161
|
+
Layer.new("#{@mapserver_url}/#{id}", path)
|
169
162
|
end
|
170
163
|
|
171
164
|
def replace_forwardslashes_with_underscores(string)
|
data/lib/gis_scraper/version.rb
CHANGED
data/lib/gis_scraper.rb
CHANGED
@@ -37,4 +37,13 @@ module GisScraper
|
|
37
37
|
@config
|
38
38
|
end
|
39
39
|
|
40
|
+
class JSONParser < Mechanize::File # shared by FeatureScraper & Layer
|
41
|
+
attr_reader :json
|
42
|
+
|
43
|
+
def initialize(uri=nil, response=nil, body=nil, code=nil)
|
44
|
+
super(uri, response, body, code)
|
45
|
+
@json = JSON.parse(body)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
40
49
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gis_scraper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.7.pre
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bruce Steedman
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-01-
|
11
|
+
date: 2016-01-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|