geo_combine 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f7557bdb34860f4f6a38081e658c51ebec60d804
-  data.tar.gz: 8d1fddf86f46fa134c5070a4c813cfdf362d1ea7
+  metadata.gz: 0370f82160756d74fe36f4c4a5e9639b8e06d461
+  data.tar.gz: e06d158e0770862c7033935c91c82bfe6b9d9f21
 SHA512:
-  metadata.gz: eaa908a3ff9e0d181236bab6a1cb3af8464e6d8912e10dc1fe6fcbb56d787ac59356777965469a8d773714566180e1657354199050b033902f2524bfb310890f
-  data.tar.gz: 5898290a91a655751dbd0bf23f5230075eeadb87546338f84c49a16c8a2fb2dd1f40bfa505a992fbb208870839ba34213e41c18da20daca98e78d8b22e5be881
+  metadata.gz: f5072b677855334cbe0716a3639394ead6c10bce24cdec3485daf5764d36df1466938247075cbf5737070a7b08fb5f1926aa618ac0c9679783c9edf8e5ea7b86
+  data.tar.gz: 26098a941f59b4c36fa0c4f8331d2c31d8575c2978e4bcba1804a9711b98a22e34d78e16dfa1a5d04da4a130f887bb91ff5bc4f0de794bb2dc2e07ffa72b93d2

data/.travis.yml

@@ -2,6 +2,6 @@ sudo: false
 language: ruby
 cache: bundler
 rvm:
-  - 2.1.10
-  - 2.2.5
-  - 2.3.1
+  - 2.2.7
+  - 2.3.4
+  - 2.4.1

data/Gemfile

@@ -3,4 +3,5 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in geo_combine.gemspec
 gemspec
 
-gem 'coveralls', require: false
+gem 'coveralls', require: false
+gem 'byebug'

data/README.md

@@ -3,12 +3,13 @@
 [![Build Status](https://travis-ci.org/OpenGeoMetadata/GeoCombine.svg?branch=master)](https://travis-ci.org/OpenGeoMetadata/GeoCombine) | [![Coverage Status](https://coveralls.io/repos/OpenGeoMetadata/GeoCombine/badge.svg?branch=master)](https://coveralls.io/r/OpenGeoMetadata/GeoCombine?branch=master)
 
 
-
-A Ruby toolkit for managing geospatial metadata
+A Ruby toolkit for managing geospatial metadata, including:
+- tasks for cloning, updating, and indexing OpenGeoMetdata metadata
+- library for converting metadata between standards
 
 ## Installation
 
-Add this line to your application's Gemfile:
+Add this line to your application's `Gemfile`:
 
 ```ruby
 gem 'geo_combine'
@@ -16,81 +17,96 @@ gem 'geo_combine'
 
 And then execute:
 
-    $ bundle
+    $ bundle install
 
 Or install it yourself as:
 
     $ gem install geo_combine
 
 ## Usage
-GeoCombine can be used as a set of rake tasks for cloning, updating, and indexing OpenGeoMetdata metdata. It can also be used as a Ruby library for converting metdata.
 
-### Transforming metadata
+### Converting metadata
 
 ```ruby
 # Create a new ISO19139 object
 > iso_metadata =  GeoCombine::Iso19139.new('./tmp/opengeometadata/edu.stanford.purl/bb/338/jh/0716/iso19139.xml')
 
-# Convert it to GeoBlacklight
+# Convert ISO to GeoBlacklight
 > iso_metadata.to_geoblacklight
 
 # Convert that to JSON
 > iso_metadata.to_geoblacklight.to_json
 
-# Convert ISO or FGDC to HTML
+# Convert ISO (or FGDC) to HTML
 > iso_metadata.to_html
 ```
 
-## Command line ##
-
-GeoCombine's tasks can be run either as rake tasks or as standalone executables.
+### OpenGeoMetadata
 
-### Clone all OpenGeoMetadata repositories
+#### Clone OpenGeoMetadata repositories locally
 
 ```sh
-$ rake geocombine:clone
+$ bundle exec rake geocombine:clone
 ```
 
+Will clone all `edu.*`,` org.*`, and `uk.*` OpenGeoMetadata repositories into `./tmp/opengeometadata`. Location of the OpenGeoMetadata repositories can be configured using the `OGM_PATH` environment variable.
+
 ```sh
-$ bundle exec geocombine clone
+$ OGM_PATH='my/custom/location' bundle exec rake geocombine:clone
 ```
 
-Will clone all edu.* OpenGeoMetadata repositories into `./tmp/opengeometadata`. Location of the OpenGeoMetadata repositories can be configured using the `OGM_PATH` environment variable.
+You can also specify a single repository:
 
 ```sh
-$ OGM_PATH='my/custom/location' rake geocombine:clone
+$ bundle exec rake geocombine:clone[edu.stanford.purl]
 ```
 
-### Pull all OpenGeoMetadata repositories
+#### Update local OpenGeoMetadata repositories
 
 ```sh
-$ rake geocombine:pull
+$ bundle exec rake geocombine:pull
 ```
 
+Runs `git pull origin master` on all cloned repositories in `./tmp/opengeometadata` (or custom path with configured environment variable `OGM_PATH`).
+
+You can also specify a single repository:
+
 ```sh
-$ bundle exec geocombine pull
+$ bundle exec rake geocombine:pull[edu.stanford.purl]
 ```
 
-Runs `git pull origin master` on all cloned repositories in `./tmp/opengeometadata` (or custom path with configured environment variable `OGM_PATH`)
+#### Index GeoBlacklight documents
 
-### Index all of the GeoBlacklight documents
+To index into Solr, GeoCombine requires a Solr instance that is running the
+[GeoBlacklight schema](https://github.com/geoblacklight/geoblacklight):
 
 ```sh
-$ rake geocombine:index
+$ bundle exec rake geocombine:index
 ```
 
+Indexes the `geoblacklight.json` files in cloned repositories to a Solr index running at http://127.0.0.1:8983/solr
+
+##### Custom Solr location
+
+Solr location can also be specified by an environment variable `SOLR_URL`.
+
 ```sh
-$ bundle exec geocombine index
+$ SOLR_URL=http://www.example.com:1234/solr/collection bundle exec rake geocombine:index
 ```
 
-Indexes all of the `geoblacklight.json` files in cloned repositories to a Solr index running at http://127.0.0.1:8983/solr
+Depending on your Solr instance's performance characteristics, you may want to
+change the [`commitWithin` parameter](https://lucene.apache.org/solr/guide/6_6/updatehandlers-in-solrconfig.html) (in milliseconds):
+
+```sh
+$ SOLR_COMMIT_WITHIN=100 bundle exec rake geocombine:index
+```
 
-#### Custom Solr location
+## Tests
 
-Solr location can also be specified by an environment variable `SOLR_URL`.
+To run the tests, use:
 
 ```sh
-$ SOLR_URL=http://www.example.com:1234/solr/collection rake geocombine:index
+$ bundle exec rake spec
 ```
 
 ## Contributing

data/geo_combine.gemspec

@@ -18,14 +18,16 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.add_dependency 'activesupport'
   spec.add_dependency 'rsolr'
+  spec.add_dependency 'net-http-persistent', '~> 2.0' # pin since faraday (rsolr) doesn't work correctly with 3.x
   spec.add_dependency 'nokogiri'
   spec.add_dependency 'json-schema'
   spec.add_dependency 'sanitize'
   spec.add_dependency 'thor'
 
-  spec.add_development_dependency "bundler", "~> 1.7"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rspec-html-matchers'
 end

data/lib/geo_combine.rb

@@ -20,7 +20,7 @@ module GeoCombine
     ##
     # Creates a new GeoCombine::Metadata object, where metadata parameter is can
     # be a File path or String of XML
-    # @param [String] metadata can be a File path 
+    # @param [String] metadata can be a File path
     # "./tmp/edu.stanford.purl/bb/338/jh/0716/iso19139.xml" or a String of XML
     # metadata
     def initialize metadata
@@ -66,6 +66,8 @@ require 'geo_combine/geoblacklight'
 require 'geo_combine/iso19139'
 require 'geo_combine/esri_open_data'
 require 'geo_combine/ckan_metadata'
+require 'geo_combine/ogp'
 
 # Require gem files
 require 'geo_combine/version'
+require 'geo_combine/railtie' if defined?(Rails)

data/lib/geo_combine/formatting.rb

@@ -15,7 +15,7 @@ module GeoCombine
     # @param [String] text
     # @return [String]
     def remove_lines(text)
-      text.gsub(/\n/, '')
+      text.delete("\n")
     end
 
     ##
@@ -25,5 +25,10 @@ module GeoCombine
     def sanitize_and_remove_lines(text)
       remove_lines(sanitize(text))
     end
+
+    # slugs should be lowercase and only have a-z, A-Z, 0-9, and -
+    def sluggify(slug)
+      slug.gsub(/[^a-zA-Z0-9\-]/, '-').gsub(/[\-]+/, '-').downcase
+    end
   end
 end

data/lib/geo_combine/geoblacklight.rb

@@ -1,3 +1,5 @@
+require 'active_support/core_ext/object/blank'
+require 'active_support/core_ext/hash/except'
 require 'open-uri'
 
 module GeoCombine
@@ -9,6 +11,16 @@ module GeoCombine
     attr_reader :metadata
 
     GEOBLACKLIGHT_VERSION = 'v1.1.0'
+    SCHEMA_JSON_URL = "https://raw.githubusercontent.com/geoblacklight/geoblacklight/#{GEOBLACKLIGHT_VERSION}/schema/geoblacklight-schema.json".freeze
+    DEPRECATED_KEYS_V1 = %w[
+      uuid
+      georss_polygon_s
+      georss_point_s
+      georss_box_s
+      dc_relation_sm
+      solr_issued_i
+      solr_bbox
+    ].freeze
 
     ##
     # Initializes a GeoBlacklight object
@@ -24,7 +36,9 @@ module GeoCombine
     # Calls metadata enhancement methods for each key, value pair in the
     # metadata hash
     def enhance_metadata
-      @metadata.each do |key, value|
+      upgrade_to_v1 if metadata['geoblacklight_version'].blank?
+
+      metadata.each do |key, value|
         translate_formats(key, value)
         enhance_subjects(key, value)
         format_proper_date(key, value)
@@ -36,15 +50,15 @@ module GeoCombine
     ##
     # Returns a string of JSON from a GeoBlacklight hash
     # @return (String)
-    def to_json
-      @metadata.to_json
+    def to_json(options = {})
+      metadata.to_json(options)
     end
 
     ##
     # Validates a GeoBlacklight-Schema json document
     # @return [Boolean]
     def valid?
-      @schema ||= JSON.parse(open("https://raw.githubusercontent.com/geoblacklight/geoblacklight/#{GEOBLACKLIGHT_VERSION}/schema/geoblacklight-schema.json").read)
+      @schema ||= JSON.parse(open(SCHEMA_JSON_URL).read)
       JSON::Validator.validate!(@schema, to_json, fragment: '#/properties/layer') &&
         dct_references_validate! &&
         spatial_validate!
@@ -54,7 +68,7 @@ module GeoCombine
     # Validate dct_references_s
     # @return [Boolean]
     def dct_references_validate!
-      return true unless metadata.key?('dct_references_s')
+      return true unless metadata.key?('dct_references_s') # TODO: shouldn't we require this field?
       begin
         ref = JSON.parse(metadata['dct_references_s'])
         raise GeoCombine::Exceptions::InvalidDCTReferences, 'dct_references must be parsed to a Hash' unless ref.is_a?(Hash)
@@ -74,43 +88,72 @@ module GeoCombine
     # Enhances the 'dc_format_s' field by translating a format type to a valid
     # GeoBlacklight-Schema format
     def translate_formats(key, value)
-      @metadata[key] = formats[value] if key == 'dc_format_s' && formats.include?(value)
+      return unless key == 'dc_format_s' && formats.include?(value)
+      metadata[key] = formats[value]
     end
 
     ##
     # Enhances the 'layer_geom_type_s' field by translating from known types
     def translate_geometry_type(key, value)
-      @metadata[key] = geometry_types[value] if key == 'layer_geom_type_s' && geometry_types.include?(value)
+      return unless key == 'layer_geom_type_s' && geometry_types.include?(value)
+      metadata[key] = geometry_types[value]
     end
 
     ##
     # Enhances the 'dc_subject_sm' field by translating subjects to ISO topic
     # categories
     def enhance_subjects(key, value)
-      @metadata[key] = value.map do |val|
+      return unless key == 'dc_subject_sm'
+      metadata[key] = value.map do |val|
         if subjects.include?(val)
           subjects[val]
         else
           val
         end
-      end if key == 'dc_subject_sm'
+      end
     end
 
     ##
     # Formats the 'layer_modified_dt' to a valid valid RFC3339 date/time string
     # and ISO8601 (for indexing into Solr)
     def format_proper_date(key, value)
-      @metadata[key] = Time.parse(value).utc.iso8601 if key == 'layer_modified_dt'
+      return unless key == 'layer_modified_dt'
+      metadata[key] = Time.parse(value).utc.iso8601
     end
 
     def fields_should_be_array(key, value)
-      @metadata[key] = [value] if should_be_array.include?(key) && !value.kind_of?(Array)
+      return unless should_be_array.include?(key) && !value.is_a?(Array)
+      metadata[key] = [value]
     end
 
     ##
     # GeoBlacklight-Schema fields that should be type Array
     def should_be_array
-      ['dc_creator_sm', 'dc_subject_sm', 'dct_spatial_sm', 'dct_temporal_sm', 'dct_isPartOf_sm']
+      %w[
+        dc_creator_sm
+        dc_subject_sm
+        dct_spatial_sm
+        dct_temporal_sm
+        dct_isPartOf_sm
+      ].freeze
+    end
+
+    ##
+    # Converts a pre-v1.0 schema into a compliant v1.0 schema
+    def upgrade_to_v1
+      metadata['geoblacklight_version'] = '1.0'
+
+      # ensure required fields
+      metadata['dc_identifier_s'] = metadata['uuid'] if metadata['dc_identifier_s'].blank?
+
+      # normalize to alphanum and - only
+      metadata['layer_slug_s'].gsub!(/[^[[:alnum:]]]+/, '-') if metadata['layer_slug_s'].present?
+
+      # remove deprecated fields
+      metadata.except!(*DEPRECATED_KEYS_V1)
+
+      # ensure we have a proper v1 record
+      valid?
     end
   end
 end

data/lib/geo_combine/ogp.rb

@@ -0,0 +1,229 @@
+require 'active_support/core_ext/object/blank'
+require 'cgi'
+
+module GeoCombine
+  # Data model for OpenGeoPortal metadata
+  class OGP
+    class InvalidMetadata < RuntimeError; end
+    include GeoCombine::Formatting
+    attr_reader :metadata
+
+    ##
+    # Initializes an OGP object for parsing
+    # @param [String] metadata a valid serialized JSON string from OGP instance
+    # @raise [InvalidMetadata]
+    def initialize(metadata)
+      @metadata = JSON.parse(metadata)
+      raise InvalidMetadata unless valid?
+    end
+
+    OGP_REQUIRED_FIELDS = %w[
+      Access
+      Institution
+      LayerDisplayName
+      LayerId
+      MaxX
+      MaxY
+      MinX
+      MinY
+      Name
+    ].freeze
+
+    ##
+    # Runs validity checks on OGP metadata to ensure fields are present
+    def valid?
+      OGP_REQUIRED_FIELDS.all? { |k| metadata[k].present? }
+    end
+
+    ##
+    # Creates and returns a Geoblacklight schema object from this metadata
+    # @return [GeoCombine::Geoblacklight]
+    def to_geoblacklight
+      GeoCombine::Geoblacklight.new(geoblacklight_terms.to_json)
+    end
+
+    ##
+    # Builds a Geoblacklight Schema type hash from Esri Open Data portal
+    # metadata
+    # @return [Hash]
+    def geoblacklight_terms
+      {
+        # Required fields
+        dc_identifier_s: identifier,
+        layer_slug_s: slug,
+        dc_title_s: metadata['LayerDisplayName'],
+        solr_geom: envelope,
+        dct_provenance_s: institution,
+        dc_rights_s: metadata['Access'],
+        geoblacklight_version: '1.0',
+
+        # Recommended fields
+        dc_description_s: metadata['Abstract'],
+        layer_geom_type_s: ogp_geom,
+        dct_references_s: references,
+        layer_id_s: "#{metadata['WorkspaceName']}:#{metadata['Name']}",
+
+        # Optional
+        dct_temporal_sm: [metadata['ContentDate']],
+        dc_format_s: ogp_formats,
+        # dct_issued_dt
+        # dc_language_s
+        dct_spatial_sm: placenames,
+        solr_year_i: year,
+        dc_publisher_s: metadata['Publisher'],
+        dc_subject_sm: subjects,
+        dc_type_s: 'Dataset'
+      }.delete_if { |_k, v| v.nil? }
+    end
+
+    def date
+      begin
+        DateTime.rfc3339(metadata['ContentDate'])
+      rescue
+        nil
+      end
+    end
+
+    def year
+      date.year unless date.nil?
+    end
+
+    ##
+    # Convert "Paper Map" to Raster, assumes all OGP "Paper Maps" have WMS
+    def ogp_geom
+      case metadata['DataType']
+      when 'Paper Map'
+        'Raster'
+      else
+        metadata['DataType']
+      end
+    end
+
+    ##
+    # OGP doesn't ship format types, so we just try and be clever here.
+    def ogp_formats
+      case metadata['DataType']
+      when 'Paper Map', 'Raster'
+        return 'GeoTIFF'
+      when 'Polygon', 'Point', 'Line'
+        return 'Shapefile'
+      else
+        raise ArgumentError, metadata['DataType']
+      end
+    end
+
+    ##
+    # Converts references to json
+    # @return [String]
+    def references
+      references_hash.to_json
+    end
+
+    ##
+    # Builds a Solr Envelope using CQL syntax
+    # @return [String]
+    def envelope
+      raise ArgumentError unless west >= -180 && west <= 180 &&
+                                 east >= -180 && east <= 180 &&
+                                 north >= -90 && north <= 90 &&
+                                 south >= -90 && south <= 90 &&
+                                 west <= east && south <= north
+      "ENVELOPE(#{west}, #{east}, #{north}, #{south})"
+    end
+
+    def subjects
+      fgdc.metadata.xpath('//themekey').map(&:text) if fgdc
+    end
+
+    def placenames
+      fgdc.metadata.xpath('//placekey').map(&:text) if fgdc
+    end
+
+    def fgdc
+      GeoCombine::Fgdc.new(metadata['FgdcText']) if metadata['FgdcText']
+    end
+
+    private
+
+    ##
+    # Builds references used for dct_references
+    # @return [Hash]
+    def references_hash
+      results = {
+        'http://www.opengis.net/def/serviceType/ogc/wfs' => location['wfs'],
+        'http://www.opengis.net/def/serviceType/ogc/wms' => location['wms'],
+        'http://schema.org/url' => location['url'],
+        download_uri => location['download']
+      }
+
+      # Handle null, "", and [""]
+      results.map { |k, v| { k => ([] << v).flatten.first } if v }
+             .flatten
+             .compact
+             .reduce({}, :merge)
+    end
+
+    def download_uri
+      return 'http://schema.org/DownloadAction' if institution == 'Harvard'
+      'http://schema.org/downloadUrl'
+    end
+
+    ##
+    # OGP "Location" field parsed
+    def location
+      JSON.parse(metadata['Location'])
+    end
+
+    def north
+      metadata['MaxY'].to_f
+    end
+
+    def south
+      metadata['MinY'].to_f
+    end
+
+    def east
+      metadata['MaxX'].to_f
+    end
+
+    def west
+      metadata['MinX'].to_f
+    end
+
+    def institution
+      metadata['Institution']
+    end
+
+    def identifier
+      CGI.escape(metadata['LayerId']) # TODO: why are we using CGI.escape?
+    end
+
+    def slug
+      name = metadata['LayerId'] || metadata['Name'] || ''
+      name = [institution, name].join('-') if institution.present? &&
+                                              !name.downcase.start_with?(institution.downcase)
+      sluggify(filter_name(name))
+    end
+
+    SLUG_BLACKLIST = %w[
+      SDE_DATA.
+      SDE.
+      SDE2.
+      GISPORTAL.GISOWNER01.
+      GISDATA.
+      MORIS.
+    ].freeze
+
+    def filter_name(name)
+      # strip out schema and usernames
+      SLUG_BLACKLIST.each do |blacklisted|
+        name.sub!(blacklisted, '')
+      end
+      unless name.size > 1
+        # use first word of title is empty name
+        name = metadata['LayerDisplayName'].split.first
+      end
+      name
+    end
+  end
+end