gis_scraper 0.1.6.pre → 0.1.7.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 19e34dd0f6e59393dff34ed06831aa6f915f8044
-  data.tar.gz: 609f5195e3f30caecf5e8d48999c1f4f449d8ae3
+  metadata.gz: d189134b4efe0805e92351f7352878d54975c1e7
+  data.tar.gz: 5dca0815dea6a1c2f3e5a142aae7b2642f872864
 SHA512:
-  metadata.gz: e37d1078cc97ca2faf7b5ca13190818485b1d2940b8c8f7185c302487eb60e7185fdf08019f843b93615e4aea3066ae92ffe7145bf43de601503695e826bfee4
-  data.tar.gz: face3402404a87c3d0346c5468b3a58781a5ad5b1d95b1bc86a6a9e1b81893e442acab0d05c7ef01253a55cc697e25170119a7c51c1c525ab98a4b24d17e8df7
+  metadata.gz: baba12b3211806d01533f0e424e34170efc88fe67c3dcb64c4573f88ad4da0fa0e28c83b9cc5abb51949c5bd2ef91b9c6bf6a5fb26bae409925e031015087a2c
+  data.tar.gz: 595016b6687eb634b8f2bc6a98b0e746061c38ea434653e5305bc31da3b3f4563ab5e8e2cc11b88e118cfec1af988d9e08efc1fc2f9a132a987df23cfa489ccf

data/lib/gis_scraper/feature_scraper.rb

@@ -1,12 +1,3 @@
-class JSONParser < Mechanize::File
-  attr_reader :json
-
-  def initialize(uri=nil, response=nil, body=nil, code=nil)
-    super(uri, response, body, code)
-    @json = JSON.parse(body)
-  end
-end
-
 class FeatureScraper
 
   attr_reader :name
@@ -14,11 +5,9 @@ class FeatureScraper
   def initialize(url)
     @url = url
     @agent = Mechanize.new
-    @agent.pluggable_parser['text/plain'] = JSONParser
-    @layer = layer # hash
-    @name = name
-    @pk = pk
-    @max = max # maxRecordCount - usually 1000
+    @agent.pluggable_parser['text/plain'] = GisScraper::JSONParser
+    @layer = layer # hash of json
+    @name, @pk, @max = name, pk, max # maxRecordCount - usually 1000
     @form = form
     @loops = loops
     @threads = GisScraper.config[:threads]
@@ -57,7 +46,7 @@ class FeatureScraper
 
   def set_query_params(loop_num = nil)
     @form.fields[0].value = where_text(loop_num)
-    loop_num ? @form.radiobuttons[4].uncheck : @form.radiobuttons[4].check # count only true
+    loop_num ? @form.radiobuttons[4].uncheck : @form.radiobuttons[4].check
     @form.fields[6].value = '*'
     @form.field_with(name: 'f').options[1].select # for JSON
   end

data/lib/gis_scraper/layer.rb

@@ -1,27 +1,15 @@
 require 'fileutils'
+require 'tmpdir'
 
 class Layer
 
-  class JSONParser < Mechanize::File
-    attr_reader :json
-
-    def initialize(uri=nil, response=nil, body=nil, code=nil)
-      super(uri, response, body, code)
-      @json = JSON.parse(body)
-    end
-  end
-
   class UnknownLayerType < StandardError; end
   class NoDatabase < StandardError; end
   class OgrMissing < StandardError; end
 
   attr_reader :type
 
-  TYPES = ['Group Layer',
-           'Feature Layer',
-           'Annotation Layer',
-           'Annotation SubLayer']
-  QUERYABLE = ['Feature Layer', 'Annotation Layer']
+  TYPE = %w(Group\ Layer Feature\ Layer Annotation\ Layer Annotation\ SubLayer)
 
   CONN = [:host, :port, :dbname, :user, :password] # PG connection options
 
@@ -31,22 +19,19 @@ class Layer
                 'esriGeometryPolyline' => 'MULTILINESTRING',
                 'esriGeometryPolygon' => 'MULTIPOLYGON'}
 
-
+  MSURL = 'MapServer'
   OGR2OGR = 'ogr2ogr -f "PostgreSQL" PG:'
 
   def initialize(url, path = nil)
     @conn_hash = CONN.zip(CONN.map { |key| GisScraper.config[key] }).to_h
     @url = url
     @output_path = output_path(path) || config_path
-    @ms_url = ms_url # map server url ending '../MapServer'
-    @id = id
+    @id, @mapserver_url = id, mapserver_url # mapserver url ends '../MapServer'
     @agent = Mechanize.new
-    @agent.pluggable_parser['text/plain'] = JSONParser
+    @agent.pluggable_parser['text/plain'] = GisScraper::JSONParser
     validate_url
     @page_json = page_json
-    @type = type
-    @name = name
-    @sub_layer_ids = sub_layer_ids
+    @type, @name, @sub_layer_ids = type, name, sub_layer_ids
   end
 
   def output_json
@@ -61,8 +46,8 @@ class Layer
 
   private
 
-  def output(format) # recurses sub-layers :)
-    QUERYABLE.any? { |l| @type == l } ? method(format) : do_sub_layers(format)
+  def output(format) # recurses sub-layers, if any (none for Annotation layers)
+    @type == 'Feature Layer' ? method(format) : do_sub_layers(format)
   end
 
   def method(format)
@@ -87,7 +72,7 @@ class Layer
     File.expand_path GisScraper.config[:output_path]
   end
 
-  def ms_url
+  def mapserver_url
     @url.split('/')[0..-2].join('/')
   end
 
@@ -97,7 +82,7 @@ class Layer
 
   def validate_url
     raise ArgumentError, 'URL must end with layer id' if  @id.to_i.to_s != @id
-    raise ArgumentError, 'Bad MapServer URL' if @ms_url[-9..-1] != 'MapServer'
+    raise ArgumentError, 'Bad MapServer URL' if @mapserver_url[-9..-1] != MSURL
   end
 
   def page_json
@@ -113,7 +98,7 @@ class Layer
   end
 
   def validate_type(type)
-    raise UnknownLayerType, type unless (TYPES.any? { |t| t == type })
+    raise UnknownLayerType, type unless (TYPE.any? { |t| t == type })
     type
   end
 
@@ -121,18 +106,26 @@ class Layer
     @page_json['subLayers'].map { |hash| hash['id'] } || []
   end
 
-  def json_data(url)
-    FeatureScraper.new(url).json_data
+  def json_data
+    FeatureScraper.new("#{@mapserver_url}/#{@id}").json_data
   end
 
   def write_json
-    File.write "#{@output_path}/#{@name}.json", json_data("#{@ms_url}/#{@id}")
+    IO.write json_path, json_data
+  end
+
+  def json_path
+    "#{@output_path}/#{@name}.json"
   end
 
   def write_to_db
-    @output_path = File.expand_path 'tmp'
-    write_json
-    `#{OGR2OGR}"#{conn}" "tmp/#{@name}.json" -nln #{table} #{srs} -nlt #{geom}`
+    @output_path = Dir.mktmpdir('gis_scraper') # prefix for identification
+    begin
+      write_json
+      `#{OGR2OGR}"#{conn}" "#{json_path}" -nln #{table} #{srs} -nlt #{geom}`
+    ensure
+      FileUtils.remove_entry @output_path
+    end
   end
 
   def geom
@@ -165,7 +158,7 @@ class Layer
   end
 
   def sub_layer(id, path)
-    Layer.new("#{@ms_url}/#{id}", path)
+    Layer.new("#{@mapserver_url}/#{id}", path)
   end
 
   def replace_forwardslashes_with_underscores(string)

data/lib/gis_scraper/version.rb

@@ -1,3 +1,3 @@
 module GisScraper
-  VERSION = '0.1.6.pre'
+  VERSION = '0.1.7.pre'
 end

data/lib/gis_scraper.rb

@@ -37,4 +37,13 @@ module GisScraper
     @config
   end
 
+  class JSONParser < Mechanize::File # shared by FeatureScraper & Layer
+    attr_reader :json
+
+    def initialize(uri=nil, response=nil, body=nil, code=nil)
+      super(uri, response, body, code)
+      @json = JSON.parse(body)
+    end
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gis_scraper
 version: !ruby/object:Gem::Version
-  version: 0.1.6.pre
+  version: 0.1.7.pre
 platform: ruby
 authors:
 - Bruce Steedman
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-01-12 00:00:00.000000000 Z
+date: 2016-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler