gis_scraper 0.1.6.pre → 0.1.7.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 19e34dd0f6e59393dff34ed06831aa6f915f8044
4
- data.tar.gz: 609f5195e3f30caecf5e8d48999c1f4f449d8ae3
3
+ metadata.gz: d189134b4efe0805e92351f7352878d54975c1e7
4
+ data.tar.gz: 5dca0815dea6a1c2f3e5a142aae7b2642f872864
5
5
  SHA512:
6
- metadata.gz: e37d1078cc97ca2faf7b5ca13190818485b1d2940b8c8f7185c302487eb60e7185fdf08019f843b93615e4aea3066ae92ffe7145bf43de601503695e826bfee4
7
- data.tar.gz: face3402404a87c3d0346c5468b3a58781a5ad5b1d95b1bc86a6a9e1b81893e442acab0d05c7ef01253a55cc697e25170119a7c51c1c525ab98a4b24d17e8df7
6
+ metadata.gz: baba12b3211806d01533f0e424e34170efc88fe67c3dcb64c4573f88ad4da0fa0e28c83b9cc5abb51949c5bd2ef91b9c6bf6a5fb26bae409925e031015087a2c
7
+ data.tar.gz: 595016b6687eb634b8f2bc6a98b0e746061c38ea434653e5305bc31da3b3f4563ab5e8e2cc11b88e118cfec1af988d9e08efc1fc2f9a132a987df23cfa489ccf
@@ -1,12 +1,3 @@
1
- class JSONParser < Mechanize::File
2
- attr_reader :json
3
-
4
- def initialize(uri=nil, response=nil, body=nil, code=nil)
5
- super(uri, response, body, code)
6
- @json = JSON.parse(body)
7
- end
8
- end
9
-
10
1
  class FeatureScraper
11
2
 
12
3
  attr_reader :name
@@ -14,11 +5,9 @@ class FeatureScraper
14
5
  def initialize(url)
15
6
  @url = url
16
7
  @agent = Mechanize.new
17
- @agent.pluggable_parser['text/plain'] = JSONParser
18
- @layer = layer # hash
19
- @name = name
20
- @pk = pk
21
- @max = max # maxRecordCount - usually 1000
8
+ @agent.pluggable_parser['text/plain'] = GisScraper::JSONParser
9
+ @layer = layer # hash of json
10
+ @name, @pk, @max = name, pk, max # maxRecordCount - usually 1000
22
11
  @form = form
23
12
  @loops = loops
24
13
  @threads = GisScraper.config[:threads]
@@ -57,7 +46,7 @@ class FeatureScraper
57
46
 
58
47
  def set_query_params(loop_num = nil)
59
48
  @form.fields[0].value = where_text(loop_num)
60
- loop_num ? @form.radiobuttons[4].uncheck : @form.radiobuttons[4].check # count only true
49
+ loop_num ? @form.radiobuttons[4].uncheck : @form.radiobuttons[4].check
61
50
  @form.fields[6].value = '*'
62
51
  @form.field_with(name: 'f').options[1].select # for JSON
63
52
  end
@@ -1,27 +1,15 @@
1
1
  require 'fileutils'
2
+ require 'tmpdir'
2
3
 
3
4
  class Layer
4
5
 
5
- class JSONParser < Mechanize::File
6
- attr_reader :json
7
-
8
- def initialize(uri=nil, response=nil, body=nil, code=nil)
9
- super(uri, response, body, code)
10
- @json = JSON.parse(body)
11
- end
12
- end
13
-
14
6
  class UnknownLayerType < StandardError; end
15
7
  class NoDatabase < StandardError; end
16
8
  class OgrMissing < StandardError; end
17
9
 
18
10
  attr_reader :type
19
11
 
20
- TYPES = ['Group Layer',
21
- 'Feature Layer',
22
- 'Annotation Layer',
23
- 'Annotation SubLayer']
24
- QUERYABLE = ['Feature Layer', 'Annotation Layer']
12
+ TYPE = %w(Group\ Layer Feature\ Layer Annotation\ Layer Annotation\ SubLayer)
25
13
 
26
14
  CONN = [:host, :port, :dbname, :user, :password] # PG connection options
27
15
 
@@ -31,22 +19,19 @@ class Layer
31
19
  'esriGeometryPolyline' => 'MULTILINESTRING',
32
20
  'esriGeometryPolygon' => 'MULTIPOLYGON'}
33
21
 
34
-
22
+ MSURL = 'MapServer'
35
23
  OGR2OGR = 'ogr2ogr -f "PostgreSQL" PG:'
36
24
 
37
25
  def initialize(url, path = nil)
38
26
  @conn_hash = CONN.zip(CONN.map { |key| GisScraper.config[key] }).to_h
39
27
  @url = url
40
28
  @output_path = output_path(path) || config_path
41
- @ms_url = ms_url # map server url ending '../MapServer'
42
- @id = id
29
+ @id, @mapserver_url = id, mapserver_url # mapserver url ends '../MapServer'
43
30
  @agent = Mechanize.new
44
- @agent.pluggable_parser['text/plain'] = JSONParser
31
+ @agent.pluggable_parser['text/plain'] = GisScraper::JSONParser
45
32
  validate_url
46
33
  @page_json = page_json
47
- @type = type
48
- @name = name
49
- @sub_layer_ids = sub_layer_ids
34
+ @type, @name, @sub_layer_ids = type, name, sub_layer_ids
50
35
  end
51
36
 
52
37
  def output_json
@@ -61,8 +46,8 @@ class Layer
61
46
 
62
47
  private
63
48
 
64
- def output(format) # recurses sub-layers :)
65
- QUERYABLE.any? { |l| @type == l } ? method(format) : do_sub_layers(format)
49
+ def output(format) # recurses sub-layers, if any (none for Annotation layers)
50
+ @type == 'Feature Layer' ? method(format) : do_sub_layers(format)
66
51
  end
67
52
 
68
53
  def method(format)
@@ -87,7 +72,7 @@ class Layer
87
72
  File.expand_path GisScraper.config[:output_path]
88
73
  end
89
74
 
90
- def ms_url
75
+ def mapserver_url
91
76
  @url.split('/')[0..-2].join('/')
92
77
  end
93
78
 
@@ -97,7 +82,7 @@ class Layer
97
82
 
98
83
  def validate_url
99
84
  raise ArgumentError, 'URL must end with layer id' if @id.to_i.to_s != @id
100
- raise ArgumentError, 'Bad MapServer URL' if @ms_url[-9..-1] != 'MapServer'
85
+ raise ArgumentError, 'Bad MapServer URL' if @mapserver_url[-9..-1] != MSURL
101
86
  end
102
87
 
103
88
  def page_json
@@ -113,7 +98,7 @@ class Layer
113
98
  end
114
99
 
115
100
  def validate_type(type)
116
- raise UnknownLayerType, type unless (TYPES.any? { |t| t == type })
101
+ raise UnknownLayerType, type unless (TYPE.any? { |t| t == type })
117
102
  type
118
103
  end
119
104
 
@@ -121,18 +106,26 @@ class Layer
121
106
  @page_json['subLayers'].map { |hash| hash['id'] } || []
122
107
  end
123
108
 
124
- def json_data(url)
125
- FeatureScraper.new(url).json_data
109
+ def json_data
110
+ FeatureScraper.new("#{@mapserver_url}/#{@id}").json_data
126
111
  end
127
112
 
128
113
  def write_json
129
- File.write "#{@output_path}/#{@name}.json", json_data("#{@ms_url}/#{@id}")
114
+ IO.write json_path, json_data
115
+ end
116
+
117
+ def json_path
118
+ "#{@output_path}/#{@name}.json"
130
119
  end
131
120
 
132
121
  def write_to_db
133
- @output_path = File.expand_path 'tmp'
134
- write_json
135
- `#{OGR2OGR}"#{conn}" "tmp/#{@name}.json" -nln #{table} #{srs} -nlt #{geom}`
122
+ @output_path = Dir.mktmpdir('gis_scraper') # prefix for identification
123
+ begin
124
+ write_json
125
+ `#{OGR2OGR}"#{conn}" "#{json_path}" -nln #{table} #{srs} -nlt #{geom}`
126
+ ensure
127
+ FileUtils.remove_entry @output_path
128
+ end
136
129
  end
137
130
 
138
131
  def geom
@@ -165,7 +158,7 @@ class Layer
165
158
  end
166
159
 
167
160
  def sub_layer(id, path)
168
- Layer.new("#{@ms_url}/#{id}", path)
161
+ Layer.new("#{@mapserver_url}/#{id}", path)
169
162
  end
170
163
 
171
164
  def replace_forwardslashes_with_underscores(string)
@@ -1,3 +1,3 @@
1
1
  module GisScraper
2
- VERSION = '0.1.6.pre'
2
+ VERSION = '0.1.7.pre'
3
3
  end
data/lib/gis_scraper.rb CHANGED
@@ -37,4 +37,13 @@ module GisScraper
37
37
  @config
38
38
  end
39
39
 
40
+ class JSONParser < Mechanize::File # shared by FeatureScraper & Layer
41
+ attr_reader :json
42
+
43
+ def initialize(uri=nil, response=nil, body=nil, code=nil)
44
+ super(uri, response, body, code)
45
+ @json = JSON.parse(body)
46
+ end
47
+ end
48
+
40
49
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gis_scraper
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.6.pre
4
+ version: 0.1.7.pre
5
5
  platform: ruby
6
6
  authors:
7
7
  - Bruce Steedman
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-01-12 00:00:00.000000000 Z
11
+ date: 2016-01-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler