gis_scraper 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 825a965fbd6ae92b8189c58b90f011ce7cb7db08e0adb2a8d579c43a830acf20
+  data.tar.gz: 8b022521994f33f1e29eb4d75a1866d693c5e6f7b414bf2458ea970363f2a49f
+SHA512:
+  metadata.gz: f63e77607228561181b993c1a5cbace0142a785747dbce0d2650c1b64301a7f6d0cb7e5aed5d2ecebfeeeb39e7236017093a9884c0af4d1c8f51d31f7199e80f
+  data.tar.gz: 5bc0e6336c552d7d87f8194ead13f9866699f79f4e71aea0175bec78efe1a3d850661badd6eed41a1cd99657996dbb17dab36c6db286665760213aecf7d98e1e

data/.gitignore

@@ -0,0 +1,6 @@
+**/.DS_Store
+/Gemfile.lock
+/pkg/
+/tmp/
+/.gitlab-ci-local
+*.gem

data/.gitlab-ci.yml

@@ -0,0 +1,30 @@
+---
+before_script:
+  - apt-get update -qq
+  - apt-get install -y postgresql postgresql-client libpq-dev gdal-bin
+  - ogr2ogr --version
+  - psql -U postgres -h $PG_HOST -d postgres -c "SELECT PostGIS_Lib_version();"
+  - pg_lsclusters
+  - ruby -v
+  - which ruby
+  - gem install bundler
+  - bundle install --jobs $(nproc)  "${FLAGS[@]}"
+  - export POSTGRES_HOST=mdillon__postgis
+
+.job_template: &job_definition
+  image: ruby:3.3.3
+
+  services:
+    # - postgres:latest  # must use host "postgres" to connect
+    - mdillon/postgis:11
+
+  variables:
+    PG_HOST: mdillon__postgis
+    POSTGRES_USER: postgres
+
+test:
+  <<: *job_definition
+  timeout: 10m
+  script:
+    - bundle exec rake spec
+    - bundle exec rubocop

data/.rspec

@@ -0,0 +1,3 @@
+--require spec_helper
+--format documentation
+--color

data/.rubocop.yml

@@ -0,0 +1,17 @@
+---
+require:
+ - rubocop-rake
+ - rubocop-rspec
+
+AllCops:
+  NewCops: enable
+
+Metrics/BlockLength:
+  Enabled: false #specs
+
+Style/HashSyntax:
+  Enabled: false # yuk Ruby 3.1
+
+RSpec/AnyInstance:
+  Enabled: false
+  

data/.ruby-version

@@ -0,0 +1 @@
+3.3.3

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## master (unreleased)
+
+## [0.0.0] - 2024-07-02
+
+* Breaking change: :arcgis_opts key renamed :arcrest_opts

data/Gemfile

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+group :test, :development do
+  gem 'rubocop-rake', '~> 0.6', require: false
+  gem 'rubocop-rspec', '~> 3.0', require: false
+end
+
+group :development do
+  gem 'bundler', '~> 2.1'
+  gem 'guard', '~> 2.18'
+  gem 'guard-rspec', '~> 4.7'
+  gem 'libnotify', '~> 0.9' # guard notifications
+  gem 'rake', '~> 13.0'
+  gem 'rspec', '~> 3.13'
+  gem 'rubocop', '~> 1.64'
+end
+
+gemspec

data/Guardfile

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+guard :rspec, cmd: 'bundle exec rspec' do
+  require 'guard/rspec/dsl'
+  dsl = Guard::RSpec::Dsl.new self
+
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_files)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 Bruce Steedman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,124 @@
+# gis_scraper Ruby Gem
+[![Gem Version](https://badge.fury.io/rb/gis_scraper.svg)](http://badge.fury.io/rb/gis_scraper)
+[![Ruby Style Guide](https://img.shields.io/badge/code_style-rubocop-brightgreen.svg)](https://github.com/rubocop/rubocop)
+
+Utility to recursively scrape ArcGIS MapServer data using the ArcGIS REST API.
+
+ArcGIS MapServer REST queries are limited to 1,000 objects or another limit. This tool makes repeated calls until all data for a given layer (and all sub-layers) are extracted. Output can be GeoJSON file format or data may be written directly to Postgres database tables in PostGIS format. GIS clients - e.g. [QGIS](https://qgis.org) - can be configured to use vector layer data from PostGIS sources.
+
+## Requirements
+
+See `.gitlab-ci.yml` file for tested Ruby versions.
+
+A Postgres database with the PostGIS extension enabled for database export.
+
+For data import to a database [GDAL](http://gdal.org) must be installed and specifically the [ogr2ogr](http://www.gdal.org/ogr2ogr.html) executable must be available in your path.
+
+## Known Limitations
+
+*NIX systems only - Linux/Mac OS X. ArcGIS MapServer data is readable directly by ArcGIS Windows clients 😉
+
+The following esri geometry types are so far supported:
+
+- esriGeometryPoint, esriGeometryMultipoint, esriGeometryLine, esriGeometryPolyline, esriGeometryPolygon
+
+Annotation layers are ignored, as are layers with no esri geometryType.
+
+Currently the JSON data for a whole layer is held in memory before being output. For large layers - e.g. >100,000 objects - this can be multiple GB of memory. Is this causes a problem for you please add a comment to [issue #4](https://gitlab.com/matzfan/gis_scraper/issues/4).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'gis_scraper'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install gis_scraper
+
+## Configuration
+
+Configuration options may be set via a hash or specified in a Yaml file. The following options are available:
+
+- ```:threads``` Scraping is multi-threaded. The number of threads to use may be set with this option (default: 8)
+- ```:output_path```    For JSON output, the path used to write files to (default: '~/Desktop')
+
+The following options are used to connect to a database:
+
+- ```:host``` (default: 'localhost')
+- ```:port``` (default:  5432)
+- ```:dbname``` (default: 'postgres')
+- ```:user``` (default: 'postgres')
+- ```:password``` (default: nil)
+
+These additional options are available when using output to a database and are applied to the ```ogr2ogr``` command:
+
+- ```:srs``` Used to overide the source spacial reference system. Currently only EPSG string format is valid - e.g. 'EPSG:3109' (default: no overide)
+
+**To set via a hash**
+
+```Ruby
+GisScraper.configure(:threads => 16) # default is 8
+```
+
+**Using a Yaml configuration file**
+
+```Ruby
+GisScraper.configure_with 'path-to-Yaml-file'
+```
+
+```Ruby
+GisScraper.config # returns the hash of configuration values
+```
+
+## Usage
+
+A LayerWriter object must be instantiated with one required arg - a Service/Layer URL (ending in an integer representing the layer number). Example:
+
+```Ruby
+writer = LayerWriter.new(url: 'https://gps.digimap.gg/arcgis/rest/services/JerseyUtilities/JerseyUtilities/MapServer/0')
+```
+An optional second argument for the output path for JSON files may be specified. If so this overides the configuration option. Example:
+```Ruby
+writer = LayerWriter.new(url: 'https://gps.digimap.gg/arcgis/rest/services/JerseyUtilities/JerseyUtilities/MapServer/0', path: '~/Desktop')
+```
+The `gis_scraper` gem uses the `arcrest` Gem [README](https://gitlab.com/matzfan/arcrest) REST API to retrieve data from ArcGIS servers. A hash of arcrest options may be passed using the :arcrest_opts key:
+```Ruby
+writer = LayerWriter.new(url: 'https://gps.digimap.gg/arcgis/rest/services/JerseyUtilities/JerseyUtilities/MapServer/0', arcrest_opts: headers: { referer: 'https://some_referrer' })
+```
+
+**JSON output**
+
+```Ruby
+writer.output_json
+```
+
+If the layer is type 'Feature Layer', a single file of JSON data will be saved (named the same as the layer). If the layer is type 'Group Layer', the sub-group structure is traversed recursively thus: Directories for each sub-group layer are created and JSON data files for each constituent feature layer written to them.
+
+**Output to a database**
+
+Valid database config options must be set. The following command will convert JSON files, create tables for each layer (& sub-layers, if any) and import the data. Table names are lowercased, prefixed '_' and have spaces replaced with undescores. If a table with the same name exists the name is appended with '_'.
+
+```Ruby
+writer.output_to_db
+```
+
+## Specification and Tests
+
+For the full specification clone this repo and run:
+
+`bundle exec rake spec`
+
+## Contributing
+
+Bug reports, pull requests (and feature requests) are welcome on GitLab at https://gitlab.com/matzfan/gis_scraper.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses)

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+# !/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'gis_scraper'
+
+require 'irb'
+IRB.start

data/bin/setup

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install

data/gis_scraper.gemspec

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'gis_scraper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'gis_scraper'
+  spec.version       = GisScraper::VERSION
+  spec.authors       = ['matzfan']
+
+  spec.summary       = 'Utility to scrape ArcGIS data'
+  spec.description   = 'Scrape ArcGIS data from REST API and export to postgres db'
+  spec.homepage      = 'https://gitlab.com/matzfan/gis_scraper'
+  spec.license = 'MIT'
+  spec.required_ruby_version = '>= 3.3.3'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://gitlab.com/matzfan/gis_scraper'
+  spec.metadata['changelog_uri'] = 'https://gitlab.com/matzfan/gis_scraper/CHANGELOG.md'
+  spec.metadata = { 'rubygems_mfa_required' => 'true' }
+
+  spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'arcrest', '~> 1.1'
+  spec.add_runtime_dependency 'parallel', '~> 1.25'
+  spec.add_runtime_dependency 'pg', '~> 1.5'
+end

data/lib/gis_scraper/feature_scraper.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# scrapes feature layers
+class FeatureScraper
+  class Ogr2ogrVersionError < StandardError; end
+
+  ESRIFIELDTYPEOID = 'esriFieldTypeOID'
+
+  attr_reader :name
+
+  def initialize(url:, arcrest_opts: {})
+    @url = url
+    @arcrest_opts = arcrest_opts # e.g. headers: { referer: '...' }
+    @layer = layer
+    @json = json
+    @name = @layer.name
+    @pk = pk
+    @max_record_count = max_record_count
+    @loops = loops
+    @threads = GisScraper.config[:threads]
+  end
+
+  def json_data
+    query_layer.merge('features' => all_features(@threads)).to_json
+  end
+
+  private
+
+  def query_layer
+    @layer.query(where: '1=1')
+  end
+
+  def layer
+    ArcREST::Layer.new(@url, @arcrest_opts)
+  end
+
+  def json
+    @layer.json
+  end
+
+  def renderer
+    @layer.drawing_info['renderer']
+  end
+
+  def pk
+    @json['fields'].select { |f| f['type'] == ESRIFIELDTYPEOID }[0]['name']
+  end
+
+  def max_record_count
+    @layer.max_record_count
+  end
+
+  def count
+    @layer.count
+  end
+
+  def features(num)
+    @layer.features(where: where_text(num))
+  end
+
+  def all_features(threads)
+    Parallel.map(0...@loops, in_threads: threads) { |n| features(n) }.flatten
+  end
+
+  def loops
+    (count.to_f / @max_record_count).ceil
+  end
+
+  def where_text(num)
+    num ? "#{pk} > #{num * @max_record_count} AND #{pk} <= #{(num + 1) * @max_record_count}" : "#{pk} > 0"
+  end
+end

data/lib/gis_scraper/layer_writer.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'shellwords'
+require 'tmpdir'
+
+# tool to write ArcGIS layer(s) to json or database output
+# rubocop:disable Metrics/ClassLength
+class LayerWriter
+  attr_reader :type
+
+  GDAL = /GDAL (\d+\.\d+\.\d+)/
+  V1_11_4 = Gem::Version.new('1.11.4') # https://trac.osgeo.org/gdal/ticket/6529
+  TABLES = "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'"
+  TYPES = ['Group ', 'Feature ', 'Annotation ', 'Annotation Sub'].freeze
+  CONN = %i[host port dbname user password].freeze
+  GEOM_TYPES = { 'esriGeometryPoint' => 'POINT',
+                 'esriGeometryMultipoint' => 'MULTIPOINT',
+                 'esriGeometryLine' => 'LINESTRING',
+                 'esriGeometryPolyline' => 'MULTILINESTRING',
+                 'esriGeometryPolygon' => 'MULTIPOLYGON' }.freeze
+  OGR = 'ogr2ogr -overwrite -f "PostgreSQL" PG:'
+
+  def initialize(url:, path: nil, arcrest_opts: {})
+    @url = url
+    @output_path = output_path(path) || config_path
+    @arcrest_opts = arcrest_opts
+    @conn = conn
+    @id = id
+    @service_url = service_url
+    @layer = layer
+    @page_json = @layer.json
+    @type = layer_type
+    @name = name
+  end
+
+  def output_json
+    output(:json)
+  end
+
+  def output_to_db
+    raise 'ogr2ogr executable missing, is GDAL installed and in your PATH?' unless (v_string = ogr2ogr?)
+    raise 'ogr2ogr version must be > 1.11.4' unless Gem::Version.new(v_string.match(GDAL)[1]) > V1_11_4
+
+    output(:db)
+  end
+
+  private
+
+  def conn
+    CONN.zip(CONN.map { |key| GisScraper.config[key] }).to_h
+  end
+
+  def output(format)
+    @type == 'Feature Layer' ? _method(format) : do_sub_layers(format) # recurses sub-layers
+  end
+
+  def _method(format)
+    format == :db ? write_to_db : write_json
+  end
+
+  def output_path(path)
+    File.expand_path(path) if path
+  end
+
+  def connection
+    PG.connect @conn
+  end
+
+  def ogr2ogr?
+    `ogr2ogr --version`
+  rescue Errno::ENOENT
+    nil
+  end
+
+  def config_path
+    File.expand_path GisScraper.config[:output_path]
+  end
+
+  def service_url
+    @url.split('/')[0..-2].join('/')
+  end
+
+  def id
+    @url.split('/').last
+  end
+
+  def layer
+    ArcREST::Layer.new(@url, @arcrest_opts)
+  end
+
+  def layer_type
+    validate_layer @page_json['type']
+  end
+
+  def validate_layer(type)
+    raise "Bad Layer type: #{type}" unless TYPES.any? { |t| type == "#{t}Layer" }
+
+    type
+  end
+
+  def name
+    @page_json['name'].tr('/', '_') # make Postgres-safe
+  end
+
+  def sub_layer_ids
+    @page_json['subLayers'].map { |hash| hash['id'] } || []
+  end
+
+  def json_data
+    FeatureScraper.new(url: "#{@service_url}/#{@id}", arcrest_opts: @arcrest_opts).json_data
+  end
+
+  def write_json
+    File.write json_path, json_data
+  end
+
+  def json_path
+    "#{@output_path}/#{@name}.json"
+  end
+
+  def write_to_db
+    @output_path = Dir.mktmpdir('gis_scraper') # prefix for identification
+    write_json
+    `#{OGR}"#{conn_str}" "#{json_path}" -nln #{table} #{srs} -nlt #{pg_geom}`
+  ensure
+    FileUtils.remove_entry @output_path
+  end
+
+  def pg_geom
+    GEOM_TYPES[geo] || raise("Unknown geom: '#{geo}' for layer #{@name}")
+  end
+
+  def geo
+    @page_json['geometryType']
+  end
+
+  def srs
+    return '' unless GisScraper.config[:srs]
+
+    "-a_srs #{GisScraper.config[:srs]}" || ''
+  end
+
+  def tables
+    connection.exec(TABLES).map { |tup| tup['table_name'] } # list of current db table names
+  end
+
+  def table
+    table_name << table_suffix
+  end
+
+  def table_name
+    Shellwords.escape(@name.downcase.tr(' ', '_')).prepend('_')
+  end
+
+  def table_suffix
+    tables.any? { |t| t == table_name } ? '_' : ''
+  end
+
+  def conn_str
+    host, port, db, user, pwd = *@conn.values
+    "dbname='#{db}' host='#{host}' port='#{port}' user='#{user}' password='#{pwd}'"
+  end
+
+  def do_sub_layers(format)
+    FileUtils.mkdir File.join(@output_path, @name) if format == :json
+    path = @output_path << "/#{@name}"
+    sub_layer_ids.each { |n| sub_layer(n, path).send(:output, format) }
+  end
+
+  def sub_layer(id, path)
+    self.class.new(url: "#{@service_url}/#{id}", path: path, arcrest_opts: @arcrest_opts) # recurse
+  end
+end
+# rubocop:enable Metrics/ClassLength

data/lib/gis_scraper/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module GisScraper
+  VERSION = '0.0.0'
+end

data/lib/gis_scraper.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'arcrest'
+require 'parallel'
+require 'pg'
+require 'yaml'
+
+require 'gis_scraper/version'
+require 'gis_scraper/feature_scraper'
+require 'gis_scraper/layer_writer'
+
+# stackoverflow.com/questions/6233124/where-to-place-access-config-file-in-gem
+module GisScraper
+  @config = { threads: 8, output_path: '~/Desktop', host: ENV.fetch('PG_HOST', 'localhost'),
+              port: 5432, dbname: 'postgres', user: 'postgres', password: nil,
+              srs: nil }
+  @valid_keys = @config.keys
+
+  def self.configure(opts = {})
+    opts.each { |k, v| @config[k.to_sym] = v if @valid_keys.include? k.to_sym }
+  end
+
+  def self.configure_with(path_to_yaml_file)
+    begin
+      config = YAML.safe_load_file path_to_yaml_file
+    rescue Errno::ENOENT
+      puts "YAML configuration file couldn't be found. Using defaults"
+      return
+    rescue Psych::SyntaxError
+      puts 'YAML configuration file contains invalid syntax. Using defaults'
+      return
+    end
+
+    configure(config)
+  end
+
+  def self.config
+    @config
+  end
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: gis_scraper
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- matzfan
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2024-07-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: arcrest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.25'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.25'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+description: Scrape ArcGIS data from REST API and export to postgres db
+email:
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".gitlab-ci.yml"
+- ".rspec"
+- ".rubocop.yml"
+- ".ruby-version"
+- CHANGELOG.md
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- gis_scraper.gemspec
+- lib/gis_scraper.rb
+- lib/gis_scraper/feature_scraper.rb
+- lib/gis_scraper/layer_writer.rb
+- lib/gis_scraper/version.rb
+homepage: https://gitlab.com/matzfan/gis_scraper
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.3.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.13
+signing_key:
+specification_version: 4
+summary: Utility to scrape ArcGIS data
+test_files: []