gis_scraper 0.1.3.pre → 0.1.4.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d095719f299da91d96069b76373e675a37a81842
-  data.tar.gz: 4d35174846e8bb601151e5a862e4ad39f40ee7e0
+  metadata.gz: 79913e4d0eaa973a3d7395a2a411253b7548aaaf
+  data.tar.gz: f9991565da85450b9cf9933ed693ab886023cda6
 SHA512:
-  metadata.gz: 69eb9967e11ff58f9de54feb8032c6097a6e7855e817acf62451b08c3df378c1c7a7c7e579f790c35fb0dacf4a1db82d13eb6d3b1c7221a3b1cdb013be705e34
-  data.tar.gz: b114a1f53b9fa97a0960e819f2ac26ae010dad9d417782451687f72f7884a4f950e0562f76ac27de09108809cda00d5c83cbe05d272d980fd393a09317869776
+  metadata.gz: 5d8f18966bcb062ffffe0aa594317322f9ffb1fea45db118f664b2df96cd896925fb444c10b6ad2dedad750f06fbb4480cdfb4f5573b1e517de98ef4467a5a91
+  data.tar.gz: 25d3d263e3be058d95849546f794fc2438cc05b7fdfde8bbc2c9068c1479c0b2fea6c53ce611031c1b1119214bb0482713ea62f4489dcbf07da71552ceb81693

data/lib/gis_scraper/layer.rb

@@ -15,7 +15,7 @@ class Layer
   class NoDatabase < StandardError; end
   class OgrMissing < StandardError; end
 
-  attr_reader :type, :id, :name
+  attr_reader :type
 
   TYPES = ['Group Layer',
            'Feature Layer',
@@ -25,19 +25,19 @@ class Layer
 
   CONN = [:host, :port, :dbname, :user, :password] # PG connection options
 
-  GEOM_TYPES = {esriGeometryPoint: 'POINT',
-                esriGeometryMultipoint: 'MULTIPOINT',
-                esriGeometryLine: 'LINESTRING',
-                esriGeometryPolyline: 'MULTILINESTRING',
-                esriGeometryPolygon: 'MULTIPOLYGON'}
+  GEOM_TYPES = {'esriGeometryPoint' => 'POINT',
+                'esriGeometryMultipoint' => 'MULTIPOINT',
+                'esriGeometryLine' => 'LINESTRING',
+                'esriGeometryPolyline' => 'MULTILINESTRING',
+                'esriGeometryPolygon' => 'MULTIPOLYGON'}
 
 
   OGR2OGR = 'ogr2ogr -f "PostgreSQL" PG:'
 
-  def initialize(url, output_path = nil)
+  def initialize(url, path = nil)
     @conn_hash = CONN.zip(CONN.map { |key| GisScraper.config[key] }).to_h
     @url = url
-    @output_path = output_path || config_path
+    @output_path = output_path(path) || config_path
     @ms_url = ms_url # map server url ending '../MapServer'
     @id = id
     @agent = Mechanize.new
@@ -46,22 +46,35 @@ class Layer
     @page_json = page_json
     @type = type
     @name = name
+    @sub_layer_ids = sub_layer_ids
   end
 
   def output_json
-    QUERYABLE.any? { |l| @type == l } ? write_json_files : process_sub_layers
+    output(:json)
   end
 
   def output_to_db
     raise OgrMissing.new, 'ogr2ogr missing, is GDAL installed?' if !ogr2ogr?
     raise NoDatabase.new, "No db connection: #{@conn_hash.inspect}" if !db?
-    @output_path = 'tmp' # write all files to the Gem's tmp dir
-    output_json
-    write_json_files_to_db_tables
+    output(:db)
   end
 
   private
 
+  def output(format) # recurses sub-layers :)
+    QUERYABLE.any? { |l| @type == l } ? method(format) : do_sub_layers(format)
+  end
+
+  def method(format)
+    return write_json if format == :json
+    return write_to_db if format == :db
+    raise "Unknown output format: #{format}"
+  end
+
+  def output_path(path)
+    File.expand_path(path) if path
+  end
+
   def db?
     PG.connect(@conn_hash) rescue nil
   end
@@ -104,31 +117,31 @@ class Layer
     type
   end
 
-  def sub_layer_id_names
-    @page_json['subLayers'] || []
+  def sub_layer_ids
+    @page_json['subLayers'].map { |hash| hash['id'] } || []
   end
 
   def json_data(url)
     FeatureScraper.new(url).json_data
   end
 
-  def write_json_files
+  def write_json
     File.write "#{@output_path}/#{@name}.json", json_data("#{@ms_url}/#{@id}")
   end
 
-  def write_json_files_to_db_tables
-    files.each do |f|
-      `#{OGR2OGR}"#{conn}" "#{f}" -nln #{base(f)} #{srs} -nlt #{geom(f)}`
-    end
+  def write_to_db
+    @output_path = 'tmp'
+    write_json
+    `#{OGR2OGR}"#{conn}" "tmp/#{@name}.json" -nln #{table} #{srs} -nlt #{geom}`
   end
 
-  def geom(file)
-    esri = esri_geom(file)
-    GEOM_TYPES[esri.to_sym] || raise("Unknown geometry type: '#{esri}'")
+  def geom
+    esri = esri_geom
+    GEOM_TYPES[esri] || raise("Unknown geometry: '#{esri}' for layer #{@name}")
   end
 
-  def esri_geom(file)
-    JSON.parse(File.read(file))['geometryType']
+  def esri_geom
+    @page_json['geometryType']
   end
 
   def srs
@@ -136,12 +149,8 @@ class Layer
     "-a_srs #{GisScraper.config[:srs]}" || ''
   end
 
-  def base(full_file_name)
-    full_file_name.split('/').last[0..-6].downcase
-  end
-
-  def files
-    Dir.glob('tmp/**/*.json')
+  def table
+    Shellwords.escape @name.downcase
   end
 
   def conn
@@ -149,21 +158,14 @@ class Layer
     "host=#{host} port=#{port} dbname=#{db} user=#{user} password=#{pwd}"
   end
 
-  def process_sub_layers
-    sub_layer_id_names.each do |hash|
-      name, id = hash['name'], hash['id']
-      path = "#{@output_path}/#{name}"
-      recurse_json sub_layer(id, path), path
-    end
-  end
-
-  def recurse_json(layer, dir)
-    FileUtils.mkdir dir
-    layer.output_json
+  def do_sub_layers(format)
+    FileUtils.mkdir File.join(@output_path, @name) if format == :json
+    path = @output_path << "/#{@name}"
+    @sub_layer_ids.each { |n| sub_layer(n, path).send(:output, format) }
   end
 
   def sub_layer(id, path)
-    Layer.new "#{@ms_url}/#{id}", path
+    Layer.new("#{@ms_url}/#{id}", path)
   end
 
   def replace_forwardslashes_with_underscores(string)

data/lib/gis_scraper/version.rb

@@ -1,3 +1,3 @@
 module GisScraper
-  VERSION = '0.1.3.pre'
+  VERSION = '0.1.4.pre'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gis_scraper
 version: !ruby/object:Gem::Version
-  version: 0.1.3.pre
+  version: 0.1.4.pre
 platform: ruby
 authors:
 - Bruce Steedman
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-12-29 00:00:00.000000000 Z
+date: 2016-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler