gis_scraper 0.1.8.pre → 0.1.9.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 13dbf8f74d525e734ed40637b4304493359c27a5
-  data.tar.gz: fc0f4fba5df9456d0fc04893ec790d3b9408e8cc
+  metadata.gz: b4673aa6d74aa17a2b9cc7edd3629955edcf2893
+  data.tar.gz: 05f54e0c85a5f5cf53ea10c6ee4ce1206e4f7bc0
 SHA512:
-  metadata.gz: 1f2e56b9e122d075263cd98763f26b9ac65984c271e13727d2a9265f0dcc997b12b39c46abaaba2dd03f4f64f3843bec6f557f653ac2cd4fa387f35725a51b65
-  data.tar.gz: 105d95e96bbd5ee51641ac8b87ba64b262523f7f36e9efa9232f5febefeb334e1c078b08a993bfe8e381a05ea44a7203d3229f80e7d8c1fa74d5543b9866ec4d
+  metadata.gz: aa65d5317c40e78c07a4007129c57079c42d7bd2389a2a98486b8970f566d8f097edb12ac1cef9811d11f9902c14ae5ecddcd7c3106be48cd2d1f22b2871d442
+  data.tar.gz: 5f9eb70359db47600488932df54bc76be6129b902e94e2ab6c9a07ed03537e07b693139ca34f63c26fd03dccc1c6220c57e1d737baaffca638c90a82ae5377d3

data/README.md

@@ -1,10 +1,10 @@
 # gis_scraper Ruby Gem
-[![Gem Version](https://badge.fury.io/rb/gis_scraper.svg)](http://badge.fury.io/rb/gis_scraper)
 [![Build status](https://secure.travis-ci.org/MatzFan/gis_scraper.svg)](http://travis-ci.org/MatzFan/gis_scraper)
+[![Gem Version](https://badge.fury.io/rb/gis_scraper.svg)](http://badge.fury.io/rb/gis_scraper)
 
 Utility to recursively scrape ArcGIS MapServer data using the ArcGIS REST API.
 
-ArcGIS MapServer REST queries are limited to 1,000 objects in some cases. This tool makes repeated calls until all data for a given layer (and all sub-layers) is extracted. Output can be JSON file format or data may be written directly to Postgres database tables in PostGIS format. GIS clients - e.g. QGIS - can be configured to use vector layer data from PostGIS sources.
+ArcGIS MapServer REST queries are limited to 1,000 objects in some cases. This tool makes repeated calls until all data for a given layer (and all sub-layers) is extracted. Output can be GeoJSON file format or data may be written directly to Postgres database tables in PostGIS format. GIS clients - e.g. QGIS - can be configured to use vector layer data from PostGIS sources.
 
 ## Requirements
 
@@ -16,12 +16,16 @@ For data import to a database [GDAL](http://gdal.org) must be installed and spec
 
 ## Known Limitations
 
-*NIX systems only - Linux/Mac OS X/Linux. ArcGIS MapServer data is readable directly by ArcGIS Windows clients.
+*NIX systems only - Linux/Mac OS X. ArcGIS MapServer data is readable directly by ArcGIS Windows clients 😉
 
-The following esri geometry types are supported:
+The following esri geometry types are so far supported:
 
 - esriGeometryPoint, esriGeometryMultipoint, esriGeometryLine, esriGeometryPolyline, esriGeometryPolygon
 
+Annotation layers are ignored, as are layers with no esri geometryType.
+
+Currently the JSON data for a whole layer is held in memory before being output. For large layers - e.g. >100,000 objects - this can be multiple GB of memory. Is this causes a problem for you please add a comment to [issue #4](https://github.com/MatzFan/gis_scraper/issues/4).
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -97,7 +101,7 @@ If the layer is type 'Feature Layer', a single file of JSON data will be saved (
 
 **Output to a database**
 
-Valid database config options must be set. The following command will convert JSON files, create tables for each layer (& sub-layers, if any) and import the data.
+Valid database config options must be set. The following command will convert JSON files, create tables for each layer (& sub-layers, if any) and import the data. Table names are lowercased, prefixed '_' and have spaces replaced with undescores. If a table with the same name exists the name is appended with '_'.
 
 ```
 layer.output_to_db

data/lib/gis_scraper/layer.rb

@@ -9,6 +9,9 @@ class Layer
 
   attr_reader :type
 
+  TABLES = "SELECT table_name FROM information_schema.tables\
+   WHERE table_schema = 'public'"
+
   TYPE = %w(Group\ Layer Feature\ Layer Annotation\ Layer Annotation\ SubLayer)
 
   CONN = [:host, :port, :dbname, :user, :password] # PG connection options
@@ -20,7 +23,7 @@ class Layer
                 'esriGeometryPolygon' => 'MULTIPOLYGON'}
 
   MSURL = 'MapServer'
-  OGR2OGR = 'ogr2ogr -f "PostgreSQL" PG:'
+  OGR = 'ogr2ogr -overwrite -f "PostgreSQL" PG:'
 
   def initialize(url, path = nil)
     @conn_hash = CONN.zip(CONN.map { |key| GisScraper.config[key] }).to_h
@@ -31,7 +34,7 @@ class Layer
     @agent.pluggable_parser['text/plain'] = GisScraper::JSONParser
     validate_url
     @page_json = page_json
-    @type, @name, @sub_layer_ids = type, name, sub_layer_ids
+    @type, @name, @sub_layer_ids, @geo = type, name, sub_layer_ids, geo
   end
 
   def output_json
@@ -40,14 +43,14 @@ class Layer
 
   def output_to_db
     raise OgrMissing.new, 'ogr2ogr missing, is GDAL installed?' if !ogr2ogr?
-    raise NoDatabase.new, "No db connection: #{@conn_hash.inspect}" if !db?
+    raise NoDatabase.new, "No db connection: #{@conn_hash.inspect}" if !conn
     output(:db)
   end
 
   private
 
   def output(format) # recurses sub-layers, if any (none for Annotation layers)
-    @type == 'Feature Layer' ? method(format) : do_sub_layers(format)
+    (@type == 'Feature Layer' && @geo) ? method(format) : do_sub_layers(format)
   end
 
   def method(format)
@@ -60,7 +63,7 @@ class Layer
     File.expand_path(path) if path
   end
 
-  def db?
+  def conn
     PG.connect(@conn_hash) rescue nil
   end
 
@@ -122,18 +125,17 @@ class Layer
     @output_path = Dir.mktmpdir('gis_scraper') # prefix for identification
     begin
       write_json
-      `#{OGR2OGR}"#{conn}" "#{json_path}" -nln #{table} #{srs} -nlt #{geom}`
+      `#{OGR}"#{c_str}" "#{json_path}" -nln #{table} #{srs} -nlt #{geom}`
     ensure
       FileUtils.remove_entry @output_path
     end
   end
 
   def geom
-    esri = esri_geom
-    GEOM_TYPES[esri] || raise("Unknown geometry: '#{esri}' for layer #{@name}")
+    GEOM_TYPES[@geo] || raise("Unknown geometry: '#{@geo}' for layer #{@name}")
   end
 
-  def esri_geom
+  def geo
     @page_json['geometryType']
   end
 
@@ -142,11 +144,23 @@ class Layer
     "-a_srs #{GisScraper.config[:srs]}" || ''
   end
 
+  def tables # list of current db table names
+    conn.exec(TABLES).map { |tup| tup['table_name'] }
+  end
+
   def table
-    '_' << Shellwords.escape(@name.downcase.gsub(' ', '_'))
+    table_name << table_suffix
   end
 
-  def conn
+  def table_name
+    Shellwords.escape(@name.downcase.gsub(' ', '_')).prepend('_')
+  end
+
+  def table_suffix
+    tables.any? { |t| t == table_name } ? '_' : ''
+  end
+
+  def c_str
     host, port, db, user, pwd = *@conn_hash.values
     "host=#{host} port=#{port} dbname=#{db} user=#{user} password=#{pwd}"
   end

data/lib/gis_scraper/version.rb

@@ -1,3 +1,3 @@
 module GisScraper
-  VERSION = '0.1.8.pre'
+  VERSION = '0.1.9.pre'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gis_scraper
 version: !ruby/object:Gem::Version
-  version: 0.1.8.pre
+  version: 0.1.9.pre
 platform: ruby
 authors:
 - Bruce Steedman
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-01-13 00:00:00.000000000 Z
+date: 2016-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler