gis_scraper 0.1.2.pre → 0.1.3.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 707ed88a572b008ddf44c0b67dd6fbd23cf78b3e
-  data.tar.gz: c5e8da833c1459487d31f81ccf0421676fe2cdf1
+  metadata.gz: d095719f299da91d96069b76373e675a37a81842
+  data.tar.gz: 4d35174846e8bb601151e5a862e4ad39f40ee7e0
 SHA512:
-  metadata.gz: aa9e8ad923993aad591c43fc2d3bdf50e9ba9500504ebff4947781a0a9641d2cc5254ca6f701aa2961c84665685196680fb67afeb5c97fcf67eda736dc1303ed
-  data.tar.gz: 51dbfe5cf8d3dbf72dc8f3ca772514857cef062f585a91067613f7cb1f59d848c6a5f338c5de8acdf70d1b6dd91247910377b56bdf016c628fa767e344267f8e
+  metadata.gz: 69eb9967e11ff58f9de54feb8032c6097a6e7855e817acf62451b08c3df378c1c7a7c7e579f790c35fb0dacf4a1db82d13eb6d3b1c7221a3b1cdb013be705e34
+  data.tar.gz: b114a1f53b9fa97a0960e819f2ac26ae010dad9d417782451687f72f7884a4f950e0562f76ac27de09108809cda00d5c83cbe05d272d980fd393a09317869776

data/.gitignore

@@ -1,3 +1,4 @@
 **/.DS_Store
 /Gemfile.lock
 /pkg/
+/tmp/

data/.travis.yml

@@ -1,5 +1,22 @@
 language: ruby
 
+addons:
+  postgresql: "9.4"
+
+services:
+  - postgresql
+
+before_script:
+  - psql -c 'create database travis_ci_test;' -U postgres
+  - psql -U postgres -c 'create extension postgis;'
+
+before_install:
+  - gem update bundler
+  # http://askubuntu.com/questions/206593/how-to-install-rgdal-on-ubuntu-12-10
+  - sudo apt-get update -qq
+  - sudo apt-get install -y aptitude
+  - sudo aptitude install -y libgdal-dev libproj-dev
+
 rvm:
   - 2.0.0
   - 2.1.6

data/README.md

@@ -2,21 +2,117 @@
 [![Gem Version](https://badge.fury.io/rb/gis_scraper.svg)](http://badge.fury.io/rb/gis_scraper)
 [![Build status](https://secure.travis-ci.org/MatzFan/gis_scraper.svg)](http://travis-ci.org/MatzFan/gis_scraper)
 
-Utility to recursively scrape ArcGIS MapServer data using REST API.
+Utility to recursively scrape ArcGIS MapServer data using the ArcGIS REST API.
 
-ArcGIS MapServer REST queries are limited to 1,000 objects in some cases. This tool makes repeated calls until all data for a given layer is extracted. It then merges the resulting JSON files into a single file. This allows GIS clients like QGIS to add a layer from a single file.
+ArcGIS MapServer REST queries are limited to 1,000 objects in some cases. This tool makes repeated calls until all data for a given layer (and all sub-layers) is extracted. Output can be JSON file format or data may be written directly to Postgres database tables in PostGIS format. GIS clients - e.g. QGIS - can be configured to use vector layer data from PostGIS sources.
 
-**Usage**
+## Requirements
 
-The executable is called 'gisget' and takes one required arg - a MapServer/Layer URL (ending in an integer representing the layer number). An optional file output path may also be specified. If omitted the file will be saved in current directory. Example:
+Ruby 2.0 or above - see Travis badge for tested Ruby versions.
 
+A Postgres database with the PostGIS extension enabled for database export.
+
+For data import to a database [GDAL](http://gdal.org) must be installed and specifically the [ogr2ogr](http://www.gdal.org/ogr2ogr.html) executable must be available in your path.
+
+## Known Limitations
+
+*NIX systems only - Linux/Mac OS X/Linux. ArcGIS MapServer data is readable directly by ArcGIS Windows clients.
+
+The following esri geometry types are supported:
+
+- esriGeometryPoint, esriGeometryMultipoint, esriGeometryLine, esriGeometryPolyline, esriGeometryPolygon
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'gis_scraper'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install gis_scraper
+
+## Configuration
+
+Configuration options may be set via a hash or specified in a Yaml file. The following options are available:
+
+- ```:threads``` Scraping is multi-threaded. The number of threads to use may be set with this option (default: 8)
+- ```:output_path```    For JSON output, the path used to write files to (default: '~/Desktop')
+
+The following options are used to connect to a database:
+
+- ```:host``` (default: 'localhost')
+- ```:port``` (default:  5432)
+- ```:dbname``` (default: 'postgres')
+- ```:user``` (default: 'postgres')
+- ```:password``` (default: nil)
+
+These additional options are available when using output to a database and are applied to the ```ogr2ogr``` command:
+
+- ```:srs``` Used to overide the source spacial reference system. Currently only EPSG string format is valid - e.g. 'EPSG:3109' (default: no overide)
+
+**To set via a hash**
+
+```Ruby
+GisScraper.configure(:threads => 16)
+```
+
+**Using a Yaml configuration file**
+
+```Ruby
+GisScraper.configure_with 'path-to-Yaml-file'
+```
+
+```Ruby
+GisScraper.config # returns the hash of configuration values
+```
+
+## Usage
+
+A Layer object must be instantiated with one required arg - a MapServer/Layer URL (ending in an integer representing the layer number). Example:
+
+```
+layer = Layer.new('http://gps.digimap.gg/arcgis/rest/services/StatesOfJersey/JerseyMappingOL/MapServer/0')
+```
+
+An optional second argument for the output path for JSON files may be specified. If so this overides the configuration option. Example:
+
+```
+layer = Layer.new('http://gps.digimap.gg/arcgis/rest/services/StatesOfJersey/JerseyMappingOL/MapServer/0', '~/Desktop')
 ```
-gisget http://gps.digimap.gg/arcgis/rest/services/StatesOfJersey/JerseyMappingOL/MapServer/0 ~/Desktop
+
+**JSON output**
+
+```
+layer.output_json
 ```
 
 If the layer is type 'Feature Layer', a single file of JSON data will be saved (named the same as the layer). If the layer is type 'Group Layer', the sub-group structure is traversed recursively thus: Directories for each sub-group layer are created and JSON data files for each constituent feature layer written to them.
 
-**Specification and Tests**
+**Output to a database**
+
+Valid database config options must be set. The following command will convert JSON files, create tables for each layer (& sub-layers, if any) and import the data.
+
+```
+layer.output_to_db
+```
+
+## Specification and Tests
+
+For the full specification clone this repo and run:
+
+`rake spec`
+
+## Contributing
+
+Bug reports, pull requests (and feature requests) are welcome on GitHub at https://github.com/MatzFan/gis_scraper.
 
-rspec spec
+## License
 
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses)

data/gis_scraper.gemspec

@@ -12,6 +12,7 @@ Gem::Specification.new do |s|
 
   s.summary       = %q{Scrapes ArcGIS data from MapServer REST API}
   s.description   = %q{Scrapes ArcGIS data from MapServer REST API}
+  s.required_ruby_version = '>= 2.0'
   s.license       = "MIT"
 
   s.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
@@ -25,4 +26,5 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency 'mechanize', '~> 2.7'
   s.add_runtime_dependency 'parallel', '~> 1.6'
+  s.add_development_dependency 'pg', '~> 0.18'
 end

data/lib/gis_scraper/layer.rb

@@ -12,6 +12,8 @@ class Layer
   end
 
   class UnknownLayerType < StandardError; end
+  class NoDatabase < StandardError; end
+  class OgrMissing < StandardError; end
 
   attr_reader :type, :id, :name
 
@@ -21,8 +23,21 @@ class Layer
            'Annotation SubLayer']
   QUERYABLE = ['Feature Layer', 'Annotation Layer']
 
-  def initialize(url, path = '.')
-    @url, @path = url, File.expand_path(path)
+  CONN = [:host, :port, :dbname, :user, :password] # PG connection options
+
+  GEOM_TYPES = {esriGeometryPoint: 'POINT',
+                esriGeometryMultipoint: 'MULTIPOINT',
+                esriGeometryLine: 'LINESTRING',
+                esriGeometryPolyline: 'MULTILINESTRING',
+                esriGeometryPolygon: 'MULTIPOLYGON'}
+
+
+  OGR2OGR = 'ogr2ogr -f "PostgreSQL" PG:'
+
+  def initialize(url, output_path = nil)
+    @conn_hash = CONN.zip(CONN.map { |key| GisScraper.config[key] }).to_h
+    @url = url
+    @output_path = output_path || config_path
     @ms_url = ms_url # map server url ending '../MapServer'
     @id = id
     @agent = Mechanize.new
@@ -33,12 +48,32 @@ class Layer
     @name = name
   end
 
-  def write
+  def output_json
     QUERYABLE.any? { |l| @type == l } ? write_json_files : process_sub_layers
   end
 
+  def output_to_db
+    raise OgrMissing.new, 'ogr2ogr missing, is GDAL installed?' if !ogr2ogr?
+    raise NoDatabase.new, "No db connection: #{@conn_hash.inspect}" if !db?
+    @output_path = 'tmp' # write all files to the Gem's tmp dir
+    output_json
+    write_json_files_to_db_tables
+  end
+
   private
 
+  def db?
+    PG.connect(@conn_hash) rescue nil
+  end
+
+  def ogr2ogr?
+    `ogr2ogr --version` rescue nil
+  end
+
+  def config_path
+    File.expand_path GisScraper.config[:output_path]
+  end
+
   def ms_url
     @url.split('/')[0..-2].join('/')
   end
@@ -78,20 +113,53 @@ class Layer
   end
 
   def write_json_files
-    File.write "#{@path}/#{@name}.json", json_data("#{@ms_url}/#{@id}")
+    File.write "#{@output_path}/#{@name}.json", json_data("#{@ms_url}/#{@id}")
+  end
+
+  def write_json_files_to_db_tables
+    files.each do |f|
+      `#{OGR2OGR}"#{conn}" "#{f}" -nln #{base(f)} #{srs} -nlt #{geom(f)}`
+    end
+  end
+
+  def geom(file)
+    esri = esri_geom(file)
+    GEOM_TYPES[esri.to_sym] || raise("Unknown geometry type: '#{esri}'")
+  end
+
+  def esri_geom(file)
+    JSON.parse(File.read(file))['geometryType']
+  end
+
+  def srs
+    return '' unless GisScraper.config[:srs]
+    "-a_srs #{GisScraper.config[:srs]}" || ''
+  end
+
+  def base(full_file_name)
+    full_file_name.split('/').last[0..-6].downcase
+  end
+
+  def files
+    Dir.glob('tmp/**/*.json')
+  end
+
+  def conn
+    host, port, db, user, pwd = *@conn_hash.values
+    "host=#{host} port=#{port} dbname=#{db} user=#{user} password=#{pwd}"
   end
 
   def process_sub_layers
     sub_layer_id_names.each do |hash|
       name, id = hash['name'], hash['id']
-      path = "#{@path}/#{name}"
-      recurse sub_layer(id, path), path
+      path = "#{@output_path}/#{name}"
+      recurse_json sub_layer(id, path), path
     end
   end
 
-  def recurse(layer, dir)
+  def recurse_json(layer, dir)
     FileUtils.mkdir dir
-    layer.write
+    layer.output_json
   end
 
   def sub_layer(id, path)

data/lib/gis_scraper/version.rb

@@ -1,3 +1,3 @@
 module GisScraper
-  VERSION = '0.1.2.pre'
+  VERSION = '0.1.3.pre'
 end

data/lib/gis_scraper.rb

@@ -1,6 +1,7 @@
 require 'yaml'
 require 'mechanize'
 require 'parallel'
+require 'pg'
 
 require 'gis_scraper/version'
 require 'gis_scraper/feature_scraper'
@@ -9,7 +10,9 @@ require 'gis_scraper/layer'
 # stackoverflow.com/questions/6233124/where-to-place-access-config-file-in-gem
 module GisScraper
 
-  @config = {threads: 8} # threads used for scraping
+  @config = {threads: 8, output_path: '~/Desktop',
+    host: 'localhost', port: 5432, dbname: 'postgres', user: 'postgres', password: nil,
+    srs: nil}
   @valid_keys = @config.keys
 
   def self.configure(opts = {})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gis_scraper
 version: !ruby/object:Gem::Version
-  version: 0.1.2.pre
+  version: 0.1.3.pre
 platform: ruby
 authors:
 - Bruce Steedman
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-12-27 00:00:00.000000000 Z
+date: 2015-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,11 +80,24 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
 description: Scrapes ArcGIS data from MapServer REST API
 email:
 - bruce.steedman@gmail.com
-executables:
-- gisget
+executables: []
 extensions: []
 extra_rdoc_files: []
 files:
@@ -97,7 +110,6 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
-- exe/gisget
 - gis_scraper.gemspec
 - lib/gis_scraper.rb
 - lib/gis_scraper/feature_scraper.rb
@@ -115,7 +127,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">"

data/exe/gisget

@@ -1,7 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'gis_scraper'
-
-start = Time.now
-Layer.new(*ARGV).write
-puts "Finished in #{Time.now - start} seconds"