gis_scraper 0.0.0 → 0.1.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 825a965fbd6ae92b8189c58b90f011ce7cb7db08e0adb2a8d579c43a830acf20
-  data.tar.gz: 8b022521994f33f1e29eb4d75a1866d693c5e6f7b414bf2458ea970363f2a49f
+SHA1:
+  metadata.gz: 7418ca46fc872f0665cf06bb5becfbe72acd1ad2
+  data.tar.gz: e3d7888a862011049f123da0941726b1231b8565
 SHA512:
-  metadata.gz: f63e77607228561181b993c1a5cbace0142a785747dbce0d2650c1b64301a7f6d0cb7e5aed5d2ecebfeeeb39e7236017093a9884c0af4d1c8f51d31f7199e80f
-  data.tar.gz: 5bc0e6336c552d7d87f8194ead13f9866699f79f4e71aea0175bec78efe1a3d850661badd6eed41a1cd99657996dbb17dab36c6db286665760213aecf7d98e1e
+  metadata.gz: d8e75fe1ac4c5a7f1a8712fb8fa1b9268b64dc79be61ed8d13f409ceb21d836b0f4f624e4119f3fb1aec4c31f9c4c302708a98f620ca9f434fac45e89215cf80
+  data.tar.gz: e35a53941f8e4788e655cff6806af3cf901748106f2f9620b6ce7951050b23ec1e3601249afb719390d95b97e013b411844230943081a1f5336de3d79a1f1a09

data/.gitignore

@@ -1,6 +1,2 @@
 **/.DS_Store
-/Gemfile.lock
-/pkg/
-/tmp/
-/.gitlab-ci-local
-*.gem
+

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+
+rvm:
+  - 2.0.0
+  - 2.1.6
+  - 2.2.3
+  - rbx-2.9
+
+script: bundle exec rspec spec

data/Gemfile

@@ -1,20 +1,3 @@
-# frozen_string_literal: true
-
 source 'https://rubygems.org'
 
-group :test, :development do
-  gem 'rubocop-rake', '~> 0.6', require: false
-  gem 'rubocop-rspec', '~> 3.0', require: false
-end
-
-group :development do
-  gem 'bundler', '~> 2.1'
-  gem 'guard', '~> 2.18'
-  gem 'guard-rspec', '~> 4.7'
-  gem 'libnotify', '~> 0.9' # guard notifications
-  gem 'rake', '~> 13.0'
-  gem 'rspec', '~> 3.13'
-  gem 'rubocop', '~> 1.64'
-end
-
 gemspec

data/Gemfile.lock

@@ -0,0 +1,62 @@
+PATH
+  remote: .
+  specs:
+    gis_scraper (0.1.0.pre)
+      mechanize (~> 2.7)
+      parallel (~> 1.6)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.2.5)
+    domain_name (0.5.25)
+      unf (>= 0.0.5, < 1.0.0)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    mechanize (2.7.3)
+      domain_name (~> 0.5, >= 0.5.1)
+      http-cookie (~> 1.0)
+      mime-types (~> 2.0)
+      net-http-digest_auth (~> 1.1, >= 1.1.1)
+      net-http-persistent (~> 2.5, >= 2.5.2)
+      nokogiri (~> 1.4)
+      ntlm-http (~> 0.1, >= 0.1.1)
+      webrobots (>= 0.0.9, < 0.2)
+    mime-types (2.99)
+    mini_portile2 (2.0.0)
+    net-http-digest_auth (1.4)
+    net-http-persistent (2.9.4)
+    nokogiri (1.6.7.1)
+      mini_portile2 (~> 2.0.0.rc2)
+    ntlm-http (0.1.1)
+    parallel (1.6.1)
+    rake (10.4.2)
+    rspec (3.3.0)
+      rspec-core (~> 3.3.0)
+      rspec-expectations (~> 3.3.0)
+      rspec-mocks (~> 3.3.0)
+    rspec-core (3.3.2)
+      rspec-support (~> 3.3.0)
+    rspec-expectations (3.3.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-mocks (3.3.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-support (3.3.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.1)
+    webrobots (0.1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.10)
+  gis_scraper!
+  rake (~> 10.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   1.10.2

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2022 Bruce Steedman
+Copyright (c) 2015 Bruce Steedman
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,124 +1,22 @@
 # gis_scraper Ruby Gem
 [![Gem Version](https://badge.fury.io/rb/gis_scraper.svg)](http://badge.fury.io/rb/gis_scraper)
-[![Ruby Style Guide](https://img.shields.io/badge/code_style-rubocop-brightgreen.svg)](https://github.com/rubocop/rubocop)
+[![Build status](https://secure.travis-ci.org/MatzFan/gis_scraper.svg)](http://travis-ci.org/MatzFan/gis_scraper)
 
-Utility to recursively scrape ArcGIS MapServer data using the ArcGIS REST API.
+Utility to recursively scrape ArcGIS MapServer data using REST API.
 
-ArcGIS MapServer REST queries are limited to 1,000 objects or another limit. This tool makes repeated calls until all data for a given layer (and all sub-layers) are extracted. Output can be GeoJSON file format or data may be written directly to Postgres database tables in PostGIS format. GIS clients - e.g. [QGIS](https://qgis.org) - can be configured to use vector layer data from PostGIS sources.
+ArcGIS MapServer REST queries are limited to 1,000 objects in some cases. This tool makes repeated calls until all data for a given layer is extracted. It then merges the resulting JSON files into a single file. This allows GIS clients like QGIS to add a layer from a single file.
 
-## Requirements
+**Usage**
 
-See `.gitlab-ci.yml` file for tested Ruby versions.
+The executable is called 'gisget' and takes one required arg - a MapServer/Layer URL (ending in an integer representing the layer number). An optional file output path may also be specified. If omitted the file will be saved in current directory. Example:
 
-A Postgres database with the PostGIS extension enabled for database export.
-
-For data import to a database [GDAL](http://gdal.org) must be installed and specifically the [ogr2ogr](http://www.gdal.org/ogr2ogr.html) executable must be available in your path.
-
-## Known Limitations
-
-*NIX systems only - Linux/Mac OS X. ArcGIS MapServer data is readable directly by ArcGIS Windows clients 😉
-
-The following esri geometry types are so far supported:
-
-- esriGeometryPoint, esriGeometryMultipoint, esriGeometryLine, esriGeometryPolyline, esriGeometryPolygon
-
-Annotation layers are ignored, as are layers with no esri geometryType.
-
-Currently the JSON data for a whole layer is held in memory before being output. For large layers - e.g. >100,000 objects - this can be multiple GB of memory. Is this causes a problem for you please add a comment to [issue #4](https://gitlab.com/matzfan/gis_scraper/issues/4).
-
-## Installation
-
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'gis_scraper'
 ```
-
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install gis_scraper
-
-## Configuration
-
-Configuration options may be set via a hash or specified in a Yaml file. The following options are available:
-
-- ```:threads``` Scraping is multi-threaded. The number of threads to use may be set with this option (default: 8)
-- ```:output_path```    For JSON output, the path used to write files to (default: '~/Desktop')
-
-The following options are used to connect to a database:
-
-- ```:host``` (default: 'localhost')
-- ```:port``` (default:  5432)
-- ```:dbname``` (default: 'postgres')
-- ```:user``` (default: 'postgres')
-- ```:password``` (default: nil)
-
-These additional options are available when using output to a database and are applied to the ```ogr2ogr``` command:
-
-- ```:srs``` Used to overide the source spacial reference system. Currently only EPSG string format is valid - e.g. 'EPSG:3109' (default: no overide)
-
-**To set via a hash**
-
-```Ruby
-GisScraper.configure(:threads => 16) # default is 8
-```
-
-**Using a Yaml configuration file**
-
-```Ruby
-GisScraper.configure_with 'path-to-Yaml-file'
-```
-
-```Ruby
-GisScraper.config # returns the hash of configuration values
-```
-
-## Usage
-
-A LayerWriter object must be instantiated with one required arg - a Service/Layer URL (ending in an integer representing the layer number). Example:
-
-```Ruby
-writer = LayerWriter.new(url: 'https://gps.digimap.gg/arcgis/rest/services/JerseyUtilities/JerseyUtilities/MapServer/0')
-```
-An optional second argument for the output path for JSON files may be specified. If so this overides the configuration option. Example:
-```Ruby
-writer = LayerWriter.new(url: 'https://gps.digimap.gg/arcgis/rest/services/JerseyUtilities/JerseyUtilities/MapServer/0', path: '~/Desktop')
-```
-The `gis_scraper` gem uses the `arcrest` Gem [README](https://gitlab.com/matzfan/arcrest) REST API to retrieve data from ArcGIS servers. A hash of arcrest options may be passed using the :arcrest_opts key:
-```Ruby
-writer = LayerWriter.new(url: 'https://gps.digimap.gg/arcgis/rest/services/JerseyUtilities/JerseyUtilities/MapServer/0', arcrest_opts: headers: { referer: 'https://some_referrer' })
-```
-
-**JSON output**
-
-```Ruby
-writer.output_json
+gisget http://gps.digimap.gg/arcgis/rest/services/StatesOfJersey/JerseyMappingOL/MapServer/0 ~/Desktop
 ```
 
 If the layer is type 'Feature Layer', a single file of JSON data will be saved (named the same as the layer). If the layer is type 'Group Layer', the sub-group structure is traversed recursively thus: Directories for each sub-group layer are created and JSON data files for each constituent feature layer written to them.
 
-**Output to a database**
-
-Valid database config options must be set. The following command will convert JSON files, create tables for each layer (& sub-layers, if any) and import the data. Table names are lowercased, prefixed '_' and have spaces replaced with undescores. If a table with the same name exists the name is appended with '_'.
-
-```Ruby
-writer.output_to_db
-```
-
-## Specification and Tests
-
-For the full specification clone this repo and run:
-
-`bundle exec rake spec`
-
-## Contributing
-
-Bug reports, pull requests (and feature requests) are welcome on GitLab at https://gitlab.com/matzfan/gis_scraper.
+**Specification and Tests**
 
-## License
+rspec spec
 
-The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses)

data/Rakefile

@@ -1,8 +1,6 @@
-# frozen_string_literal: true
-
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task default: :spec
+task :default => :spec

data/bin/console

@@ -1,9 +1,14 @@
-# frozen_string_literal: true
-
-# !/usr/bin/env ruby
+#!/usr/bin/env ruby
 
 require 'bundler/setup'
 require 'gis_scraper'
 
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
 require 'irb'
 IRB.start

data/bin/gisget

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+start = Time.now
+Layer.new(*ARGV).write
+puts "Finished in #{Time.now - start} seconds"

data/gis_scraper.gemspec

@@ -1,32 +1,28 @@
-# frozen_string_literal: true
-
-lib = File.expand_path('lib', __dir__)
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'gis_scraper/version'
 
-Gem::Specification.new do |spec|
-  spec.name          = 'gis_scraper'
-  spec.version       = GisScraper::VERSION
-  spec.authors       = ['matzfan']
+Gem::Specification.new do |s|
+  s.name          = 'gis_scraper'
+  s.version       = GisScraper::VERSION
+  s.authors       = ['Bruce Steedman']
+  s.email         = ['bruce.steedman@gmail.com']
 
-  spec.summary       = 'Utility to scrape ArcGIS data'
-  spec.description   = 'Scrape ArcGIS data from REST API and export to postgres db'
-  spec.homepage      = 'https://gitlab.com/matzfan/gis_scraper'
-  spec.license = 'MIT'
-  spec.required_ruby_version = '>= 3.3.3'
+  s.summary       = %q{Scrapes ArcGIS data from MapServer REST API}
+  s.description   = %q{Scrapes ArcGIS data from MapServer REST API}
+  s.license       = "MIT"
 
-  spec.metadata['homepage_uri'] = spec.homepage
-  spec.metadata['source_code_uri'] = 'https://gitlab.com/matzfan/gis_scraper'
-  spec.metadata['changelog_uri'] = 'https://gitlab.com/matzfan/gis_scraper/CHANGELOG.md'
-  spec.metadata = { 'rubygems_mfa_required' => 'true' }
+  s.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
+  s.bindir        = 'exe'
+  s.executables   = s.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  s.require_paths = ['lib']
 
-  spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
-  spec.bindir        = 'exe'
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ['lib']
+  s.add_development_dependency 'bundler', '~> 1.10'
+  s.add_development_dependency 'rake', '~> 10.0'
+  s.add_development_dependency 'rspec', '~> 3.0'
 
-  spec.add_runtime_dependency 'arcrest', '~> 1.1'
-  spec.add_runtime_dependency 'parallel', '~> 1.25'
-  spec.add_runtime_dependency 'pg', '~> 1.5'
+  s.add_runtime_dependency 'mechanize', '~> 2.7'
+  s.add_runtime_dependency 'parallel', '~> 1.6'
 end

data/lib/gis_scraper/feature_scraper.rb

@@ -1,72 +1,82 @@
-# frozen_string_literal: true
+class JSONParser < Mechanize::File
+  attr_reader :json
 
-# scrapes feature layers
-class FeatureScraper
-  class Ogr2ogrVersionError < StandardError; end
+  def initialize(uri=nil, response=nil, body=nil, code=nil)
+    super(uri, response, body, code)
+    @json = JSON.parse(body)
+  end
+end
 
-  ESRIFIELDTYPEOID = 'esriFieldTypeOID'
+class FeatureScraper
 
   attr_reader :name
 
-  def initialize(url:, arcrest_opts: {})
+  def initialize(url)
     @url = url
-    @arcrest_opts = arcrest_opts # e.g. headers: { referer: '...' }
-    @layer = layer
-    @json = json
-    @name = @layer.name
+    @agent = Mechanize.new
+    @agent.pluggable_parser['text/plain'] = JSONParser
+    @layer = layer # hash
+    @name = name
     @pk = pk
-    @max_record_count = max_record_count
+    @max = max # maxRecordCount - usually 1000
+    @form = form
     @loops = loops
     @threads = GisScraper.config[:threads]
   end
 
   def json_data
-    query_layer.merge('features' => all_features(@threads)).to_json
+    data(0).merge({'features' => features(@threads)}).to_json
   end
 
   private
 
-  def query_layer
-    @layer.query(where: '1=1')
-  end
-
   def layer
-    ArcREST::Layer.new(@url, @arcrest_opts)
+    @agent.get(@url + '?f=pjson').json
   end
 
-  def json
-    @layer.json
+  def name
+    @layer['name']
   end
 
-  def renderer
-    @layer.drawing_info['renderer']
+  def pk
+    @layer['fields'].select { |f| f['type'] == 'esriFieldTypeOID' }[0]['name']
   end
 
-  def pk
-    @json['fields'].select { |f| f['type'] == ESRIFIELDTYPEOID }[0]['name']
+  def max
+    @layer['maxRecordCount'].to_i
   end
 
-  def max_record_count
-    @layer.max_record_count
+  def form
+    @agent.get(@url + '/query').forms.first
   end
 
   def count
-    @layer.count
+    set_query_params
+    @form.submit(@form.buttons[1]).json['count'].to_i
   end
 
-  def features(num)
-    @layer.features(where: where_text(num))
+  def set_query_params(loop_num = nil)
+    @form.fields[0].value = where_text(loop_num)
+    loop_num ? @form.radiobuttons[4].uncheck : @form.radiobuttons[4].check # count only true
+    @form.fields[6].value = '*'
+    @form.field_with(name: 'f').options[1].select # for JSON
   end
 
-  def all_features(threads)
-    Parallel.map(0...@loops, in_threads: threads) { |n| features(n) }.flatten
+  def data(n)
+    set_query_params(n)
+    @form.submit(@form.buttons[1]).json
+  end
+
+  def features(t)
+    Parallel.map(0...@loops, in_threads: t) { |n| data(n)['features'] }.flatten
   end
 
   def loops
-    (count.to_f / @max_record_count).ceil
+    (count.to_f/@max).ceil
   end
 
-  def where_text(num)
-    num ? "#{pk} > #{num * @max_record_count} AND #{pk} <= #{(num + 1) * @max_record_count}" : "#{pk} > 0"
+  def where_text(n)
+    n ? "#{pk} > #{n * @max} AND #{pk} <= #{(n + 1) * @max}" : "#{pk} > 0"
   end
+
 end

data/lib/gis_scraper/layer.rb

@@ -0,0 +1,105 @@
+require 'fileutils'
+
+class Layer
+
+  class JSONParser < Mechanize::File
+    attr_reader :json
+
+    def initialize(uri=nil, response=nil, body=nil, code=nil)
+      super(uri, response, body, code)
+      @json = JSON.parse(body)
+    end
+  end
+
+  class UnknownLayerType < StandardError; end
+
+  attr_reader :type, :id, :name
+
+  TYPES = ['Group Layer',
+           'Feature Layer',
+           'Annotation Layer',
+           'Annotation SubLayer']
+  QUERYABLE = ['Feature Layer', 'Annotation Layer']
+
+  def initialize(url, path = '.')
+    @url, @path = url, File.expand_path(path)
+    @ms_url = ms_url # map server url ending '../MapServer'
+    @id = id
+    @agent = Mechanize.new
+    @agent.pluggable_parser['text/plain'] = JSONParser
+    validate_url
+    @page_json = page_json
+    @type = type
+    @name = name
+  end
+
+  def write
+    QUERYABLE.any? { |l| @type == l } ? write_json_files : process_sub_layers
+  end
+
+  private
+
+  def ms_url
+    @url.split('/')[0..-2].join('/')
+  end
+
+  def id
+    @url.split('/').last
+  end
+
+  def validate_url
+    raise ArgumentError, 'URL must end with layer id' if  @id.to_i.to_s != @id
+    raise ArgumentError, 'Bad MapServer URL' if @ms_url[-9..-1] != 'MapServer'
+  end
+
+  def page_json
+    @agent.get(@url + '?f=pjson').json
+  end
+
+  def type
+    validate_type @page_json['type']
+  end
+
+  def name
+    replace_forwardslashes_with_underscores @page_json['name']
+  end
+
+  def validate_type(type)
+    raise UnknownLayerType, type unless (TYPES.any? { |t| t == type })
+    type
+  end
+
+  def sub_layer_id_names
+    @page_json['subLayers'] || []
+  end
+
+  def json_data(url)
+    FeatureScraper.new(url).json_data
+  end
+
+  def write_json_files
+    File.write "#{@path}/#{@name}.json", json_data("#{@ms_url}/#{@id}")
+  end
+
+  def process_sub_layers
+    sub_layer_id_names.each do |hash|
+      name, id = hash['name'], hash['id']
+      path = "#{@path}/#{name}"
+      recurse sub_layer(id, path), path
+    end
+  end
+
+  def recurse(layer, dir)
+    FileUtils.mkdir dir
+    layer.write
+  end
+
+  def sub_layer(id, path)
+    Layer.new "#{@ms_url}/#{id}", path
+  end
+
+  def replace_forwardslashes_with_underscores(string)
+    string.gsub /\//, '_'
+  end
+
+end

data/lib/gis_scraper/version.rb

@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 module GisScraper
-  VERSION = '0.0.0'
+  VERSION = '0.1.0.pre'
 end

data/lib/gis_scraper.rb

@@ -1,28 +1,24 @@
-# frozen_string_literal: true
-
-require 'arcrest'
-require 'parallel'
-require 'pg'
-require 'yaml'
-
 require 'gis_scraper/version'
 require 'gis_scraper/feature_scraper'
-require 'gis_scraper/layer_writer'
+require 'gis_scraper/layer'
+
+require 'yaml'
+require 'mechanize'
+require 'parallel'
 
 # stackoverflow.com/questions/6233124/where-to-place-access-config-file-in-gem
 module GisScraper
-  @config = { threads: 8, output_path: '~/Desktop', host: ENV.fetch('PG_HOST', 'localhost'),
-              port: 5432, dbname: 'postgres', user: 'postgres', password: nil,
-              srs: nil }
+
+  @config = {threads: 8} # threads used for scraping
   @valid_keys = @config.keys
 
   def self.configure(opts = {})
-    opts.each { |k, v| @config[k.to_sym] = v if @valid_keys.include? k.to_sym }
+    opts.each { |k,v| @config[k.to_sym] = v if @valid_keys.include? k.to_sym }
   end
 
   def self.configure_with(path_to_yaml_file)
     begin
-      config = YAML.safe_load_file path_to_yaml_file
+      config = YAML::load(IO.read(path_to_yaml_file))
     rescue Errno::ENOENT
       puts "YAML configuration file couldn't be found. Using defaults"
       return
@@ -37,4 +33,5 @@ module GisScraper
   def self.config
     @config
   end
+
 end

metadata

@@ -1,87 +1,113 @@
 --- !ruby/object:Gem::Specification
 name: gis_scraper
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.1.0.pre
 platform: ruby
 authors:
-- matzfan
-autorequire:
+- Bruce Steedman
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-07-02 00:00:00.000000000 Z
+date: 2015-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: arcrest
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
-  type: :runtime
+        version: '1.10'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.10'
 - !ruby/object:Gem::Dependency
-  name: parallel
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.25'
+        version: '2.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.25'
+        version: '2.7'
 - !ruby/object:Gem::Dependency
-  name: pg
+  name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
-description: Scrape ArcGIS data from REST API and export to postgres db
+        version: '1.6'
+description: Scrapes ArcGIS data from MapServer REST API
 email:
+- bruce.steedman@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".gitlab-ci.yml"
 - ".rspec"
-- ".rubocop.yml"
-- ".ruby-version"
-- CHANGELOG.md
+- ".travis.yml"
 - Gemfile
-- Guardfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/console
+- bin/gisget
 - bin/setup
 - gis_scraper.gemspec
 - lib/gis_scraper.rb
 - lib/gis_scraper/feature_scraper.rb
-- lib/gis_scraper/layer_writer.rb
+- lib/gis_scraper/layer.rb
 - lib/gis_scraper/version.rb
-homepage: https://gitlab.com/matzfan/gis_scraper
+homepage: 
 licenses:
 - MIT
-metadata:
-  rubygems_mfa_required: 'true'
-post_install_message:
+metadata: {}
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -89,15 +115,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.3.3
+      version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.5.13
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
 specification_version: 4
-summary: Utility to scrape ArcGIS data
+summary: Scrapes ArcGIS data from MapServer REST API
 test_files: []

data/.gitlab-ci.yml

@@ -1,30 +0,0 @@
----
-before_script:
-  - apt-get update -qq
-  - apt-get install -y postgresql postgresql-client libpq-dev gdal-bin
-  - ogr2ogr --version
-  - psql -U postgres -h $PG_HOST -d postgres -c "SELECT PostGIS_Lib_version();"
-  - pg_lsclusters
-  - ruby -v
-  - which ruby
-  - gem install bundler
-  - bundle install --jobs $(nproc)  "${FLAGS[@]}"
-  - export POSTGRES_HOST=mdillon__postgis
-
-.job_template: &job_definition
-  image: ruby:3.3.3
-
-  services:
-    # - postgres:latest  # must use host "postgres" to connect
-    - mdillon/postgis:11
-
-  variables:
-    PG_HOST: mdillon__postgis
-    POSTGRES_USER: postgres
-
-test:
-  <<: *job_definition
-  timeout: 10m
-  script:
-    - bundle exec rake spec
-    - bundle exec rubocop

data/.rubocop.yml

@@ -1,17 +0,0 @@
----
-require:
- - rubocop-rake
- - rubocop-rspec
-
-AllCops:
-  NewCops: enable
-
-Metrics/BlockLength:
-  Enabled: false #specs
-
-Style/HashSyntax:
-  Enabled: false # yuk Ruby 3.1
-
-RSpec/AnyInstance:
-  Enabled: false
-  

data/.ruby-version

@@ -1 +0,0 @@
-3.3.3

data/CHANGELOG.md

@@ -1,5 +0,0 @@
-## master (unreleased)
-
-## [0.0.0] - 2024-07-02
-
-* Breaking change: :arcgis_opts key renamed :arcrest_opts

data/Guardfile

@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-guard :rspec, cmd: 'bundle exec rspec' do
-  require 'guard/rspec/dsl'
-  dsl = Guard::RSpec::Dsl.new self
-
-  rspec = dsl.rspec
-  watch(rspec.spec_helper) { rspec.spec_dir }
-  watch(rspec.spec_files)
-
-  # Ruby files
-  ruby = dsl.ruby
-  dsl.watch_spec_files_for(ruby.lib_files)
-end

data/lib/gis_scraper/layer_writer.rb

@@ -1,175 +0,0 @@
-# frozen_string_literal: true
-
-require 'fileutils'
-require 'shellwords'
-require 'tmpdir'
-
-# tool to write ArcGIS layer(s) to json or database output
-# rubocop:disable Metrics/ClassLength
-class LayerWriter
-  attr_reader :type
-
-  GDAL = /GDAL (\d+\.\d+\.\d+)/
-  V1_11_4 = Gem::Version.new('1.11.4') # https://trac.osgeo.org/gdal/ticket/6529
-  TABLES = "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'"
-  TYPES = ['Group ', 'Feature ', 'Annotation ', 'Annotation Sub'].freeze
-  CONN = %i[host port dbname user password].freeze
-  GEOM_TYPES = { 'esriGeometryPoint' => 'POINT',
-                 'esriGeometryMultipoint' => 'MULTIPOINT',
-                 'esriGeometryLine' => 'LINESTRING',
-                 'esriGeometryPolyline' => 'MULTILINESTRING',
-                 'esriGeometryPolygon' => 'MULTIPOLYGON' }.freeze
-  OGR = 'ogr2ogr -overwrite -f "PostgreSQL" PG:'
-
-  def initialize(url:, path: nil, arcrest_opts: {})
-    @url = url
-    @output_path = output_path(path) || config_path
-    @arcrest_opts = arcrest_opts
-    @conn = conn
-    @id = id
-    @service_url = service_url
-    @layer = layer
-    @page_json = @layer.json
-    @type = layer_type
-    @name = name
-  end
-
-  def output_json
-    output(:json)
-  end
-
-  def output_to_db
-    raise 'ogr2ogr executable missing, is GDAL installed and in your PATH?' unless (v_string = ogr2ogr?)
-    raise 'ogr2ogr version must be > 1.11.4' unless Gem::Version.new(v_string.match(GDAL)[1]) > V1_11_4
-
-    output(:db)
-  end
-
-  private
-
-  def conn
-    CONN.zip(CONN.map { |key| GisScraper.config[key] }).to_h
-  end
-
-  def output(format)
-    @type == 'Feature Layer' ? _method(format) : do_sub_layers(format) # recurses sub-layers
-  end
-
-  def _method(format)
-    format == :db ? write_to_db : write_json
-  end
-
-  def output_path(path)
-    File.expand_path(path) if path
-  end
-
-  def connection
-    PG.connect @conn
-  end
-
-  def ogr2ogr?
-    `ogr2ogr --version`
-  rescue Errno::ENOENT
-    nil
-  end
-
-  def config_path
-    File.expand_path GisScraper.config[:output_path]
-  end
-
-  def service_url
-    @url.split('/')[0..-2].join('/')
-  end
-
-  def id
-    @url.split('/').last
-  end
-
-  def layer
-    ArcREST::Layer.new(@url, @arcrest_opts)
-  end
-
-  def layer_type
-    validate_layer @page_json['type']
-  end
-
-  def validate_layer(type)
-    raise "Bad Layer type: #{type}" unless TYPES.any? { |t| type == "#{t}Layer" }
-
-    type
-  end
-
-  def name
-    @page_json['name'].tr('/', '_') # make Postgres-safe
-  end
-
-  def sub_layer_ids
-    @page_json['subLayers'].map { |hash| hash['id'] } || []
-  end
-
-  def json_data
-    FeatureScraper.new(url: "#{@service_url}/#{@id}", arcrest_opts: @arcrest_opts).json_data
-  end
-
-  def write_json
-    File.write json_path, json_data
-  end
-
-  def json_path
-    "#{@output_path}/#{@name}.json"
-  end
-
-  def write_to_db
-    @output_path = Dir.mktmpdir('gis_scraper') # prefix for identification
-    write_json
-    `#{OGR}"#{conn_str}" "#{json_path}" -nln #{table} #{srs} -nlt #{pg_geom}`
-  ensure
-    FileUtils.remove_entry @output_path
-  end
-
-  def pg_geom
-    GEOM_TYPES[geo] || raise("Unknown geom: '#{geo}' for layer #{@name}")
-  end
-
-  def geo
-    @page_json['geometryType']
-  end
-
-  def srs
-    return '' unless GisScraper.config[:srs]
-
-    "-a_srs #{GisScraper.config[:srs]}" || ''
-  end
-
-  def tables
-    connection.exec(TABLES).map { |tup| tup['table_name'] } # list of current db table names
-  end
-
-  def table
-    table_name << table_suffix
-  end
-
-  def table_name
-    Shellwords.escape(@name.downcase.tr(' ', '_')).prepend('_')
-  end
-
-  def table_suffix
-    tables.any? { |t| t == table_name } ? '_' : ''
-  end
-
-  def conn_str
-    host, port, db, user, pwd = *@conn.values
-    "dbname='#{db}' host='#{host}' port='#{port}' user='#{user}' password='#{pwd}'"
-  end
-
-  def do_sub_layers(format)
-    FileUtils.mkdir File.join(@output_path, @name) if format == :json
-    path = @output_path << "/#{@name}"
-    sub_layer_ids.each { |n| sub_layer(n, path).send(:output, format) }
-  end
-
-  def sub_layer(id, path)
-    self.class.new(url: "#{@service_url}/#{id}", path: path, arcrest_opts: @arcrest_opts) # recurse
-  end
-end
-# rubocop:enable Metrics/ClassLength