gis_scraper 0.1.9.pre → 0.1.10.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b4673aa6d74aa17a2b9cc7edd3629955edcf2893
-  data.tar.gz: 05f54e0c85a5f5cf53ea10c6ee4ce1206e4f7bc0
+  metadata.gz: 6c95a3f2a8c7e50cf4df5e6f0f363d660e19c5a8
+  data.tar.gz: 5b6e929eec3af5ad0b767d34239de3d00d8e4d67
 SHA512:
-  metadata.gz: aa65d5317c40e78c07a4007129c57079c42d7bd2389a2a98486b8970f566d8f097edb12ac1cef9811d11f9902c14ae5ecddcd7c3106be48cd2d1f22b2871d442
-  data.tar.gz: 5f9eb70359db47600488932df54bc76be6129b902e94e2ab6c9a07ed03537e07b693139ca34f63c26fd03dccc1c6220c57e1d737baaffca638c90a82ae5377d3
+  metadata.gz: d13788b449d9c87a009816b48ef89038a7b686307ed68a915b02059580d6189ca946e3889485ee26f50d2e2cb1b296504cb32353d8f4461c7435e5e58fe86a27
+  data.tar.gz: cb6ec2b8c1b747a32e799dc7a905d7824b3e7f5b2a6382230ad7d4017ef2a13773a7ee4c63af900edacbf02454d0d18c45989f9a1894dcd0310cf1100807e540

data/.travis.yml

@@ -1,26 +1,21 @@
 language: ruby
-
-addons:
-  postgresql: "9.4"
+rvm:
+  - 2.1
+  - 2.2
 
 services:
   - postgresql
 
-before_script:
-  - psql -c 'create database travis_ci_test;' -U postgres
-  - psql -U postgres -c 'create extension postgis;'
+addons:
+  postgresql: "9.4"
 
 before_install:
   - gem update bundler
-  # http://askubuntu.com/questions/206593/how-to-install-rgdal-on-ubuntu-12-10
-  - sudo apt-get update -qq
-  - sudo apt-get install -y aptitude
-  - sudo aptitude install -y libgdal-dev libproj-dev
 
-rvm:
-  - 2.0.0
-  - 2.1.6
-  - 2.2.3
-  - rbx-2.9
+before_script:
+  - sudo rm /etc/apt/sources.list.d/ubuntugis-stable-source.list # https://github.com/travis-ci/travis-ci/issues/2401
+  - sudo apt-get update -qq
+  - sudo apt-get install gdal-bin -y
+  - psql -c "create extension postgis" -U $USER -d $USER # default db name is same as user
 
 script: bundle exec rspec spec

data/README.md

@@ -79,22 +79,22 @@ GisScraper.config # returns the hash of configuration values
 
 ## Usage
 
-A Layer object must be instantiated with one required arg - a MapServer/Layer URL (ending in an integer representing the layer number). Example:
+A LayerWriter object must be instantiated with one required arg - a Service/Layer URL (ending in an integer representing the layer number). Example:
 
 ```
-layer = Layer.new('http://gps.digimap.gg/arcgis/rest/services/StatesOfJersey/JerseyMappingOL/MapServer/0')
+writer = LayerWriter.new('http://gps.digimap.gg/arcgis/rest/services/StatesOfJersey/JerseyMappingOL/MapServer/0')
 ```
 
 An optional second argument for the output path for JSON files may be specified. If so this overides the configuration option. Example:
 
 ```
-layer = Layer.new('http://gps.digimap.gg/arcgis/rest/services/StatesOfJersey/JerseyMappingOL/MapServer/0', '~/Desktop')
+writer = LayerWriter.new('http://gps.digimap.gg/arcgis/rest/services/StatesOfJersey/JerseyMappingOL/MapServer/0', '~/Desktop')
 ```
 
 **JSON output**
 
 ```
-layer.output_json
+writer.output_json
 ```
 
 If the layer is type 'Feature Layer', a single file of JSON data will be saved (named the same as the layer). If the layer is type 'Group Layer', the sub-group structure is traversed recursively thus: Directories for each sub-group layer are created and JSON data files for each constituent feature layer written to them.
@@ -104,7 +104,7 @@ If the layer is type 'Feature Layer', a single file of JSON data will be saved (
 Valid database config options must be set. The following command will convert JSON files, create tables for each layer (& sub-layers, if any) and import the data. Table names are lowercased, prefixed '_' and have spaces replaced with undescores. If a table with the same name exists the name is appended with '_'.
 
 ```
-layer.output_to_db
+writer.output_to_db
 ```
 
 ## Specification and Tests

data/Rakefile

@@ -3,4 +3,4 @@ require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/gis_scraper.gemspec

@@ -10,21 +10,21 @@ Gem::Specification.new do |s|
   s.authors       = ['Bruce Steedman']
   s.email         = ['bruce.steedman@gmail.com']
 
-  s.summary       = %q{Scrapes ArcGIS data from MapServer REST API}
-  s.description   = %q{Scrapes ArcGIS data from MapServer REST API}
+  s.summary       = 'Scrapes ArcGIS data from MapServer REST API'
+  s.description   = 'Scrapes ArcGIS data from MapServer REST API'
   s.required_ruby_version = '>= 2.0'
-  s.license       = "MIT"
+  s.license = 'MIT'
 
-  s.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
+  s.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   s.bindir        = 'exe'
   s.executables   = s.files.grep(%r{^exe/}) { |f| File.basename(f) }
   s.require_paths = ['lib']
 
   s.add_development_dependency 'bundler', '~> 1.11'
   s.add_development_dependency 'rake', '~> 10.0'
-  s.add_development_dependency 'rspec', '~> 3.0'
+  s.add_development_dependency 'rspec', '~> 3.4'
 
-  s.add_runtime_dependency 'mechanize', '~> 2.7'
-  s.add_runtime_dependency 'parallel', '~> 1.6'
+  s.add_runtime_dependency 'arcrest', '~> 0.0.2'
+  s.add_runtime_dependency 'parallel', '~> 1.9'
   s.add_development_dependency 'pg', '~> 0.18'
 end

data/lib/gis_scraper.rb

@@ -1,27 +1,27 @@
 require 'yaml'
-require 'mechanize'
+require 'arcrest'
+# require 'mechanize'
 require 'parallel'
 require 'pg'
 
 require 'gis_scraper/version'
 require 'gis_scraper/feature_scraper'
-require 'gis_scraper/layer'
+require 'gis_scraper/layer_writer'
 
 # stackoverflow.com/questions/6233124/where-to-place-access-config-file-in-gem
 module GisScraper
-
-  @config = {threads: 8, output_path: '~/Desktop',
-    host: 'localhost', port: 5432, dbname: 'postgres', user: 'postgres', password: nil,
-    srs: nil}
+  @config = { threads: 8, output_path: '~/Desktop', host: 'localhost',
+              port: 5432, dbname: 'postgres', user: 'postgres', password: nil,
+              srs: nil }
   @valid_keys = @config.keys
 
   def self.configure(opts = {})
-    opts.each { |k,v| @config[k.to_sym] = v if @valid_keys.include? k.to_sym }
+    opts.each { |k, v| @config[k.to_sym] = v if @valid_keys.include? k.to_sym }
   end
 
   def self.configure_with(path_to_yaml_file)
     begin
-      config = YAML::load(IO.read(path_to_yaml_file))
+      config = YAML.load(IO.read(path_to_yaml_file))
     rescue Errno::ENOENT
       puts "YAML configuration file couldn't be found. Using defaults"
       return
@@ -37,13 +37,13 @@ module GisScraper
     @config
   end
 
-  class JSONParser < Mechanize::File # shared by FeatureScraper & Layer
-    attr_reader :json
-
-    def initialize(uri=nil, response=nil, body=nil, code=nil)
-      super(uri, response, body, code)
-      @json = JSON.parse(body)
-    end
-  end
+  # shared by FeatureScraper & Layer
+  # class JSONParser < Mechanize::File
+  #   attr_reader :json
 
+  #   def initialize(uri = nil, response = nil, body = nil, code = nil)
+  #     super(uri, response, body, code)
+  #     @json = JSON.parse(body)
+  #   end
+  # end
 end

data/lib/gis_scraper/feature_scraper.rb

@@ -1,71 +1,89 @@
+# scrapes feature layers
 class FeatureScraper
+  API_CALL_LIMIT = 1000
+  STRING = 'esriFieldTypeString'.freeze
+  VARCHAR_MAX_SIZE = 10_485_760 # max size for PostgreSQL VARCHAR
 
   attr_reader :name
 
   def initialize(url)
     @url = url
-    @agent = Mechanize.new
-    @agent.pluggable_parser['text/plain'] = GisScraper::JSONParser
-    @layer = layer # hash of json
-    @name, @pk, @max = name, pk, max # maxRecordCount - usually 1000
-    @form = form
+    @layer = layer
+    @json = json
+    @name = name
+    @pk = pk
+    @max = max # maxRecordCount - usually 1000
     @loops = loops
     @threads = GisScraper.config[:threads]
   end
 
   def json_data
-    data(0).merge({'features' => features(@threads)}).to_json
+    query_without_features.merge('features' => all_features(@threads)).to_json
   end
 
   private
 
+  def query_without_features # check_field_length not needed ogr2ogr >= 1.11.5
+    check_field_length @layer.query(where: '1=0')
+  end
+
   def layer
-    @agent.get(@url + '?f=pjson').json
+    ArcREST::Layer.new(@url)
+  end
+
+  def json
+    @layer.json
   end
 
   def name
-    @layer['name']
+    @layer.name
+  end
+
+  def renderer
+    @layer.drawing_info['renderer']
   end
 
   def pk
-    @layer['fields'].select { |f| f['type'] == 'esriFieldTypeOID' }[0]['name']
+    @json['fields'].select { |f| f['type'] == 'esriFieldTypeOID' }[0]['name']
   end
 
   def max
-    @layer['maxRecordCount'].to_i
+    @layer.max_record_count || API_CALL_LIMIT
   end
 
-  def form
-    @agent.get(@url + '/query').forms.first
+  def count
+    @layer.count
   end
 
-  def count
-    set_query_params
-    @form.submit(@form.buttons[1]).json['count'].to_i
+  def features(n)
+    @layer.features(where: where_text(n))
+  end
+
+  def check_field_length(hash) # https://trac.osgeo.org/gdal/ticket/6529
+    hash.merge check_fields(hash['fields'])
   end
 
-  def set_query_params(loop_num = nil)
-    @form.fields[0].value = where_text(loop_num)
-    loop_num ? @form.radiobuttons[4].uncheck : @form.radiobuttons[4].check
-    @form.fields[6].value = '*'
-    @form.field_with(name: 'f').options[1].select # for JSON
+  def check_fields(fields)
+    { 'fields' => fields.map { |f| f['type'] == STRING ? esri_string(f) : f } }
   end
 
-  def data(n)
-    set_query_params(n)
-    @form.submit(@form.buttons[1]).json
+  def esri_string(fields)
+    Hash[fields.map { |k, v| [k, k == 'length' ? truncate(v) : v] }] # nice :)
   end
 
-  def features(t)
-    Parallel.map(0...@loops, in_threads: t) { |n| data(n)['features'] }.flatten
+  def truncate(length)
+    length > VARCHAR_MAX_SIZE ? 0 : length
+  end
+
+  def all_features(threads)
+    Parallel.map(0...@loops, in_threads: threads) { |n| features(n) }.flatten
   end
 
   def loops
-    (count.to_f/@max).ceil
+    (count.to_f / @max).ceil
   end
 
   def where_text(n)
     n ? "#{pk} > #{n * @max} AND #{pk} <= #{(n + 1) * @max}" : "#{pk} > 0"
   end
-
 end

data/lib/gis_scraper/{layer.rb → layer_writer.rb}

@@ -1,8 +1,9 @@
 require 'fileutils'
 require 'tmpdir'
+require 'shellwords'
 
-class Layer
-
+# tool to write ArcGIS layer(s) to json or database output
+class LayerWriter
   class UnknownLayerType < StandardError; end
   class NoDatabase < StandardError; end
   class OgrMissing < StandardError; end
@@ -10,31 +11,33 @@ class Layer
   attr_reader :type
 
   TABLES = "SELECT table_name FROM information_schema.tables\
-   WHERE table_schema = 'public'"
+   WHERE table_schema = 'public'".freeze
 
-  TYPE = %w(Group\ Layer Feature\ Layer Annotation\ Layer Annotation\ SubLayer)
+  TYPE = ['Group Layer', 'Feature Layer', 'Annotation Layer',
+          'Annotation SubLayer'].freeze
 
-  CONN = [:host, :port, :dbname, :user, :password] # PG connection options
+  CONN = [:host, :port, :dbname, :user, :password].freeze
 
-  GEOM_TYPES = {'esriGeometryPoint' => 'POINT',
-                'esriGeometryMultipoint' => 'MULTIPOINT',
-                'esriGeometryLine' => 'LINESTRING',
-                'esriGeometryPolyline' => 'MULTILINESTRING',
-                'esriGeometryPolygon' => 'MULTIPOLYGON'}
+  GEOM_TYPES = { 'esriGeometryPoint' => 'POINT',
+                 'esriGeometryMultipoint' => 'MULTIPOINT',
+                 'esriGeometryLine' => 'LINESTRING',
+                 'esriGeometryPolyline' => 'MULTILINESTRING',
+                 'esriGeometryPolygon' => 'MULTIPOLYGON' }.freeze
 
-  MSURL = 'MapServer'
-  OGR = 'ogr2ogr -overwrite -f "PostgreSQL" PG:'
+  OGR = 'ogr2ogr -overwrite -f "PostgreSQL" PG:'.freeze
 
   def initialize(url, path = nil)
     @conn_hash = CONN.zip(CONN.map { |key| GisScraper.config[key] }).to_h
     @url = url
     @output_path = output_path(path) || config_path
-    @id, @mapserver_url = id, mapserver_url # mapserver url ends '../MapServer'
-    @agent = Mechanize.new
-    @agent.pluggable_parser['text/plain'] = GisScraper::JSONParser
-    validate_url
+    @id = id
+    @service_url = service_url
+    @layer = layer
     @page_json = page_json
-    @type, @name, @sub_layer_ids, @geo = type, name, sub_layer_ids, geo
+    @type = type
+    @name = name
+    @sub_layer_ids = sub_layer_ids
+    @geo = geo
   end
 
   def output_json
@@ -42,8 +45,8 @@ class Layer
   end
 
   def output_to_db
-    raise OgrMissing.new, 'ogr2ogr missing, is GDAL installed?' if !ogr2ogr?
-    raise NoDatabase.new, "No db connection: #{@conn_hash.inspect}" if !conn
+    raise OgrMissing.new, 'ogr2ogr missing, is GDAL installed?' unless ogr2ogr?
+    raise NoDatabase.new, "No db connection: #{@conn_hash.inspect}" unless conn
     output(:db)
   end
 
@@ -75,7 +78,7 @@ class Layer
     File.expand_path GisScraper.config[:output_path]
   end
 
-  def mapserver_url
+  def service_url
     @url.split('/')[0..-2].join('/')
   end
 
@@ -83,13 +86,12 @@ class Layer
     @url.split('/').last
   end
 
-  def validate_url
-    raise ArgumentError, 'URL must end with layer id' if  @id.to_i.to_s != @id
-    raise ArgumentError, 'Bad MapServer URL' if @mapserver_url[-9..-1] != MSURL
+  def layer
+    ArcREST::Layer.new @url
   end
 
   def page_json
-    @agent.get(@url + '?f=pjson').json
+    @layer.json
   end
 
   def type
@@ -101,7 +103,7 @@ class Layer
   end
 
   def validate_type(type)
-    raise UnknownLayerType, type unless (TYPE.any? { |t| t == type })
+    raise UnknownLayerType, type unless TYPE.any? { |t| t == type }
     type
   end
 
@@ -110,7 +112,7 @@ class Layer
   end
 
   def json_data
-    FeatureScraper.new("#{@mapserver_url}/#{@id}").json_data
+    FeatureScraper.new("#{@service_url}/#{@id}").json_data
   end
 
   def write_json
@@ -125,14 +127,14 @@ class Layer
     @output_path = Dir.mktmpdir('gis_scraper') # prefix for identification
     begin
       write_json
-      `#{OGR}"#{c_str}" "#{json_path}" -nln #{table} #{srs} -nlt #{geom}`
+      `#{OGR}"#{conn_str}" "#{json_path}" -nln #{table} #{srs} -nlt #{geom}`
     ensure
       FileUtils.remove_entry @output_path
     end
   end
 
   def geom
-    GEOM_TYPES[@geo] || raise("Unknown geometry: '#{@geo}' for layer #{@name}")
+    GEOM_TYPES[@geo] || raise("Unknown geom: '#{@geo}' for layer #{@name}")
   end
 
   def geo
@@ -153,14 +155,14 @@ class Layer
   end
 
   def table_name
-    Shellwords.escape(@name.downcase.gsub(' ', '_')).prepend('_')
+    Shellwords.escape(@name.downcase.tr(' ', '_')).prepend('_')
   end
 
   def table_suffix
     tables.any? { |t| t == table_name } ? '_' : ''
   end
 
-  def c_str
+  def conn_str
     host, port, db, user, pwd = *@conn_hash.values
     "host=#{host} port=#{port} dbname=#{db} user=#{user} password=#{pwd}"
   end
@@ -172,11 +174,10 @@ class Layer
   end
 
   def sub_layer(id, path)
-    Layer.new("#{@mapserver_url}/#{id}", path)
+    self.class.new("#{@service_url}/#{id}", path)
   end
 
   def replace_forwardslashes_with_underscores(string)
-    string.gsub /\//, '_'
+    string.tr('/', '_')
   end
-
 end

data/lib/gis_scraper/version.rb

@@ -1,3 +1,3 @@
 module GisScraper
-  VERSION = '0.1.9.pre'
+  VERSION = '0.1.10.pre'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gis_scraper
 version: !ruby/object:Gem::Version
-  version: 0.1.9.pre
+  version: 0.1.10.pre
 platform: ruby
 authors:
 - Bruce Steedman
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-01-15 00:00:00.000000000 Z
+date: 2016-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,42 +44,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.4'
 - !ruby/object:Gem::Dependency
-  name: mechanize
+  name: arcrest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: 0.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: 0.0.2
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.9'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -113,7 +113,7 @@ files:
 - gis_scraper.gemspec
 - lib/gis_scraper.rb
 - lib/gis_scraper/feature_scraper.rb
-- lib/gis_scraper/layer.rb
+- lib/gis_scraper/layer_writer.rb
 - lib/gis_scraper/version.rb
 homepage: 
 licenses:
@@ -135,7 +135,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Scrapes ArcGIS data from MapServer REST API