geo_combine 0.0.5 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 23a7b7ef3140404f61f2c0143e25556163178bb1
-  data.tar.gz: f776389c9604fd18cbdf958c2f68b2efd4181fd2
+  metadata.gz: 8627401efabab03d06f210ee160e7e09c7223398
+  data.tar.gz: dcdc93c201e689dc2c9b8245ac2076617af34e39
 SHA512:
-  metadata.gz: 1b9ecf8ae11b1e211665e5d90921b32a5f53c7d7e07bfbb5b7d7ab46c6cf27e3e759230b94ff6c8197ba4c38b8a1eb11e299dc3e5b879c15907db01f9982e728
-  data.tar.gz: 12e7c094382bc5ea0fe803c2fe7b1aef653102d3082ceb4b9f1068e3d4a1124a571b3b3189030ac5378bcdc3f84cbdf86e3159085460adf565931a2b4b68d419
+  metadata.gz: 9c957f3b783684f18c18c3265349c7efa4db3aa12094f573e9d4d9e5a2366f3464e390add7a12734213ca1801c5ef3d0d820a8139fdce47c07ba99e037f71e5f
+  data.tar.gz: e6bedee395328db1deed473721231e7a5d678e0bfa2a9b038e3f7de596e99622cc7dff7b4dc2d6db14d2d5734339d01fa9d5a462bfbe4363872afe87c98ecb48

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/README.md

@@ -29,7 +29,7 @@ GeoCombine can be used as a set of rake tasks for cloning, updating, and indexin
 
 ```ruby
 # Create a new ISO19139 object
-> iso_metadata =  GeoCombine::Iso19139.new('./tmp/edu.stanford.purl/bb/338/jh/0716/iso19139.xml')
+> iso_metadata =  GeoCombine::Iso19139.new('./tmp/opengeometadata/edu.stanford.purl/bb/338/jh/0716/iso19139.xml')
 
 # Convert it to GeoBlacklight
 > iso_metadata.to_geoblacklight
@@ -41,13 +41,25 @@ GeoCombine can be used as a set of rake tasks for cloning, updating, and indexin
 > iso_metadata.to_html
 ```
 
+## Command line ##
+
+GeoCombine's tasks can be run either as rake tasks or as standalone executables.
+
 ### Clone all OpenGeoMetadata repositories
 
 ```sh
 $ rake geocombine:clone
 ```
 
-Will clone all edu.* OpenGeoMetadata repositories into `./tmp`
+```sh
+$ bundle exec geocombine clone
+```
+
+Will clone all edu.* OpenGeoMetadata repositories into `./tmp/opengeometadata`. Location of the OpenGeoMetadata repositories can be configured using the `OGM_PATH` environment variable.
+
+```sh
+$ OGM_PATH='my/custom/location' rake geocombine:clone
+```
 
 ### Pull all OpenGeoMetadata repositories
 
@@ -55,7 +67,11 @@ Will clone all edu.* OpenGeoMetadata repositories into `./tmp`
 $ rake geocombine:pull
 ```
 
-Runs `git pull origin master` on all cloned repositories in `./tmp`
+```sh
+$ bundle exec geocombine pull
+```
+
+Runs `git pull origin master` on all cloned repositories in `./tmp/opengeometadata` (or custom path with configured environment variable `OGM_PATH`)
 
 ### Index all of the GeoBlacklight documents
 
@@ -63,7 +79,11 @@ Runs `git pull origin master` on all cloned repositories in `./tmp`
 $ rake geocombine:index
 ```
 
-Indexs all of the `geoblacklight.xml` files in cloned repositories to a Solr index running at http://127.0.0.1:8983/solr
+```sh
+$ bundle exec geocombine index
+```
+
+Indexs all of the `geoblacklight.json` files in cloned repositories to a Solr index running at http://127.0.0.1:8983/solr
 
 ## Contributing
 

data/bin/geocombine

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'geo_combine/cli'
+
+GeoCombine::CLI.start(ARGV)

data/geo_combine.gemspec

@@ -20,8 +20,12 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'rsolr'
   spec.add_dependency 'nokogiri'
+  spec.add_dependency 'json-schema'
+  spec.add_dependency 'sanitize'
+  spec.add_dependency 'thor'
 
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rspec-html-matchers'
 end

data/lib/geo_combine/cli.rb

@@ -0,0 +1,25 @@
+require 'thor'
+require 'rake'
+
+root = Gem::Specification.find_by_name('geo_combine').gem_dir
+tasks = File.join(root, 'lib/tasks/*.rake')
+Dir.glob(tasks).each { |r| load r }
+
+module GeoCombine
+  class CLI < Thor
+    desc 'clone', 'Clone all OpenGeoMetadata repositories'
+    def clone
+      Rake::Task['geocombine:clone'].invoke
+    end
+
+    desc 'pull', '"git pull" OpenGeoMetadata repositories'
+    def pull
+      Rake::Task['geocombine:pull'].invoke
+    end
+
+    desc "index", "Index all of the GeoBlacklight documents"
+    def index
+      Rake::Task['geocombine:index'].invoke
+    end
+  end
+end

data/lib/geo_combine/esri_open_data.rb

@@ -0,0 +1,97 @@
+module GeoCombine
+  # Data model for ESRI's open data portal metadata
+  class EsriOpenData
+    include GeoCombine::Formatting
+    attr_reader :metadata
+
+    ##
+    # Initializes an EsriOpenData object for parsing
+    # @param [String] metadata a valid serialized JSON string from an ESRI Open
+    # Data portal
+    def initialize(metadata)
+      @metadata = JSON.parse(metadata)
+      @geometry = @metadata['extent']['coordinates']
+    end
+
+    ##
+    # Creates and returns a Geoblacklight schema object from this metadata
+    # @return [GeoCombine::Geoblacklight]
+    def to_geoblacklight
+      GeoCombine::Geoblacklight.new(geoblacklight_terms.to_json)
+    end
+
+    ##
+    # Builds a Geoblacklight Schema type hash from Esri Open Data portal
+    # metadata
+    # @return [Hash]
+    def geoblacklight_terms
+      {
+        uuid: @metadata['id'],
+        dc_identifier_s: @metadata['id'],
+        dc_title_s: @metadata['name'],
+        dc_description_s: sanitize_and_remove_lines(@metadata['description']),
+        dc_rights_s: 'Public',
+        dct_provenance_s: @metadata['owner'],
+        dct_references_s: references,
+        georss_box_s: georss_box,
+        # layer_id_s is used for describing a layer id for a web serivce (WMS, WFS) but is still a required field
+        layer_id_s: '',
+        layer_geom_type_s: @metadata['geometry_type'],
+        layer_modified_dt: @metadata['updated_at'],
+        layer_slug_s: @metadata['id'],
+        solr_geom: envelope,
+        # solr_year_i: '', No equivalent in Esri Open Data metadata
+        dc_subject_sm: @metadata['tags']
+      }
+    end
+
+    ##
+    # Converts references to json
+    # @return [String]
+    def references
+      references_hash.to_json
+    end
+
+    ##
+    # Builds references used for dct_references
+    # @return [Hash]
+    def references_hash
+      {
+        'http://schema.org/url' => @metadata['landing_page'],
+        'http://resources.arcgis.com/en/help/arcgis-rest-api' => @metadata['url']
+      }
+    end
+
+    ##
+    # Builds a GeoRSS box
+    # @return [String]
+    def georss_box
+      "#{south} #{west} #{north} #{east}"
+    end
+
+    ##
+    # Builds a Solr Envelope using CQL syntax
+    # @return [String]
+    def envelope
+      "ENVELOPE(#{west}, #{east}, #{north}, #{south})"
+    end
+
+    private
+
+    def north
+      @geometry[1][1]
+    end
+
+    def south
+      @geometry[0][1]
+    end
+
+    def east
+      @geometry[1][0]
+    end
+
+    def west
+      @geometry[0][0]
+    end
+  end
+end

data/lib/geo_combine/formats.rb

@@ -0,0 +1,11 @@
+module GeoCombine
+  ##
+  # Translation dictionary for mime-type to valid GeoBlacklight-Schema formats
+  module Formats
+    def formats
+      {
+        'application/x-esri-shapefile' => 'Shapefile'
+      }
+    end
+  end
+end

data/lib/geo_combine/formatting.rb

@@ -0,0 +1,29 @@
+module GeoCombine
+  ##
+  # Mixin used for formatting metadata fields
+  module Formatting
+    ##
+    # Sanitizes html from a text input
+    # @param [String] text
+    # @return [String]
+    def sanitize(text)
+      Sanitize.fragment(text)
+    end
+
+    ##
+    # Removes line breaks from a text input
+    # @param [String] text
+    # @return [String]
+    def remove_lines(text)
+      text.gsub(/\n/, '')
+    end
+
+    ##
+    # Sanitizes and removes lines from a text block
+    # @param [String] text
+    # @return [String]
+    def sanitize_and_remove_lines(text)
+      remove_lines(sanitize(text))
+    end
+  end
+end

data/lib/geo_combine/geoblacklight.rb

@@ -1,25 +1,91 @@
 module GeoCombine
-  class Geoblacklight < Metadata
+  class Geoblacklight
+    include GeoCombine::Formats
+    include GeoCombine::Subjects
+    include GeoCombine::GeometryTypes
+
+    attr_reader :metadata
+
+    ##
+    # Initializes a GeoBlacklight object
+    # @param [String] metadata be a valid JSON string document in
+    # GeoBlacklight-Schema
+    # @param [Hash] fields enhancements to metadata that are merged with @metadata
+    def initialize(metadata, fields = {})
+      @metadata = JSON.parse(metadata).merge(fields)
+    end
+
+    ##
+    # Calls metadata enhancement methods for each key, value pair in the
+    # metadata hash
+    def enhance_metadata
+      @metadata.each do |key, value|
+        translate_formats(key, value)
+        enhance_subjects(key, value)
+        format_proper_date(key, value)
+        fields_should_be_array(key, value)
+        translate_geometry_type(key, value)
+      end
+    end
 
     ##
     # Returns a string of JSON from a GeoBlacklight hash
     # @return (String)
     def to_json
-      to_hash.to_json
+      @metadata.to_json
     end
 
     ##
-    # Returns a hash from a GeoBlacklight object
-    # @return (Hash)
-    def to_hash
-      hash = {}
-      @metadata.css('field').each do |field|
-        (hash[field.attributes['name'].value] ||= []) << field.children.text
-      end
-      hash.collect do |key, value|
-        hash[key] = value.count > 1 ? { key => value } : { key => value[0] }
-      end
-      hash
+    # Validates a GeoBlacklight-Schema json document
+    # @return [Boolean]
+    def valid?
+      schema = JSON.parse(File.read(File.join(File.dirname(__FILE__), '../schema/geoblacklight-schema.json')))
+      JSON::Validator.validate!(schema, to_json, validate_schema: true)
+    end
+
+    private
+
+    ##
+    # Enhances the 'dc_format_s' field by translating a format type to a valid
+    # GeoBlacklight-Schema format
+    def translate_formats(key, value)
+      @metadata[key] = formats[value] if key == 'dc_format_s' && formats.include?(value)
+    end
+
+    ##
+    # Enhances the 'layer_geom_type_s' field by translating from known types
+    def translate_geometry_type(key, value)
+      @metadata[key] = geometry_types[value] if key == 'layer_geom_type_s' && geometry_types.include?(value)
+    end
+
+    ##
+    # Enhances the 'dc_subject_sm' field by translating subjects to ISO topic
+    # categories
+    def enhance_subjects(key, value)
+      @metadata[key] = value.map do |val|
+        if subjects.include?(val)
+          subjects[val]
+        else
+          val
+        end
+      end if key == 'dc_subject_sm'
+    end
+
+    ##
+    # Formats the 'layer_modified_dt' to a valid valid RFC3339 date/time string
+    # and ISO8601 (for indexing into Solr)
+    def format_proper_date(key, value)
+      @metadata[key] = Time.parse(value).utc.iso8601 if key == 'layer_modified_dt'
+    end
+
+    def fields_should_be_array(key, value)
+      @metadata[key] = [value] if should_be_array.include?(key) && !value.kind_of?(Array)
+    end
+
+    ##
+    # GeoBlacklight-Schema fields that should be type Array
+    def should_be_array
+      ['dc_creator_sm', 'dc_subject_sm', 'dct_spatial_sm', 'dct_temporal_sm', 'dct_isPartOf_sm']
     end
   end
 end

data/lib/geo_combine/geometry_types.rb

@@ -0,0 +1,11 @@
+module GeoCombine
+  module GeometryTypes
+    def geometry_types
+      {
+        'esriGeometryPoint' => 'Point',
+        'esriGeometryPolygon' => 'Polygon',
+        'esriGeometryPolyline' => 'Line'
+      }
+    end
+  end
+end

data/lib/geo_combine/subjects.rb

@@ -0,0 +1,29 @@
+module GeoCombine
+  ##
+  # Translation dictionary to ISO topics
+  module Subjects
+    def subjects
+      {
+        'farming' => 'Farming',
+        'biota' => 'Biology and Ecology',
+        'climatologyMeteorologyAtmosphere' => 'Climatology, Meteorology and Atmosphere',
+        'boundaries' => 'Boundaries',
+        'elevation' => 'Elevation',
+        'environment' => 'Environment',
+        'geoscientificInformation' => 'Geoscientific Information',
+        'health' => 'Health',
+        'imageryBaseMapsEarthCover' => 'Imagery and Base Maps',
+        'intelligenceMilitary' => 'Military',
+        'inlandWaters' => 'Inland Waters',
+        'location' => 'Location',
+        'oceans' => 'Oceans',
+        'planningCadastre' => 'Planning and Cadastral',
+        'structure' => 'Structure',
+        'transportation' => 'Transportation',
+        'utilitiesCommunication' => 'Utilities and Communication',
+        'society' => 'Society',
+        'economy' => 'Economy'
+      }
+    end
+  end
+end

data/lib/geo_combine/version.rb

@@ -1,3 +1,3 @@
 module GeoCombine
-  VERSION = '0.0.5'
+  VERSION = '0.1.0'
 end

data/lib/geo_combine.rb

@@ -1,4 +1,7 @@
 require 'nokogiri'
+require 'json'
+require 'json-schema'
+require 'sanitize'
 
 module GeoCombine
 
@@ -28,9 +31,12 @@ module GeoCombine
 
     ##
     # Perform an XSLT tranformation on metadata using an object's XSL
-    # @return [GeoCombine::Geoblacklight] the data transformed into geoblacklight schema, returned as a GeoCombine::Geoblacklight
-    def to_geoblacklight
-      GeoCombine::Geoblacklight.new(xsl_geoblacklight.transform(@metadata))
+    # @return fields additional GeoBlacklight fields to be passed to
+    # GeoCombine::Geoblacklight on its instantiation
+    # @return [GeoCombine::Geoblacklight] the data transformed into
+    # geoblacklight schema, returned as a GeoCombine::Geoblacklight
+    def to_geoblacklight fields = {}
+      GeoCombine::Geoblacklight.new(xsl_geoblacklight.apply_to(@metadata), fields)
     end
 
     ##
@@ -42,7 +48,19 @@ module GeoCombine
   end
 end
 
+# Require translation mixins
+require 'geo_combine/formats'
+require 'geo_combine/subjects'
+require 'geo_combine/geometry_types'
+
+# Require helper mixins
+require 'geo_combine/formatting'
+
+# Require additional classes
 require 'geo_combine/fgdc'
 require 'geo_combine/geoblacklight'
 require 'geo_combine/iso19139'
+require 'geo_combine/esri_open_data'
+
+# Require gem files
 require 'geo_combine/version'

data/lib/schema/geoblacklight-schema.json

@@ -0,0 +1,169 @@
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "Schema for GeoBlacklight as implemented for Solr 4.10+. See http://journal.code4lib.org/articles/9710 for more details. Note that the Solr schema uses dynamic typing based on the suffix of the field name. For example, _s denotes a string where _sm denotes a multi-valued string (array of strings).",
+    "id": "http://geoblacklight.org/schema",
+    "title": "GeoBlacklight Schema",
+    "required": [
+        "uuid",
+        "dc_identifier_s",
+        "dc_title_s",
+        "dc_description_s",
+        "dc_rights_s",
+        "dct_provenance_s",
+        "dct_references_s",
+        "georss_box_s",
+        "layer_id_s",
+        "layer_geom_type_s",
+        "layer_modified_dt",
+        "layer_slug_s",
+        "solr_geom",
+        "solr_year_i"
+    ],
+    "type": "object",
+    "properties": {
+        "uuid": { 
+            "type": "string", 
+            "description": "Unique identifier for layer that is globally unique."
+
+        },
+        "dc_identifier_s": {
+            "type": "string",
+            "description": "Unique identifier for layer. May be same as UUID but may be an alternate identifier."
+        },
+        "dc_title_s": { 
+            "type": "string", 
+            "description": "Title for the layer."
+
+        },
+        "dc_description_s": { 
+            "type": "string", 
+            "description": "Description for the layer."
+
+        },
+        "dc_rights_s": { 
+            "type": "string",
+            "enum": ["Public", "Restricted"], 
+            "description": "Access rights for the layer."
+
+        },
+        "dct_provenance_s": { 
+            "type": "string", 
+            "description": "Institution who holds the layer."
+
+        },
+        "dct_references_s": { 
+            "type": "string", 
+            "description": "JSON hash for external resources, where each key is a URI for the protocol or format and the value is the URL to the resource."
+
+        },
+        "georss_box_s": { 
+            "type": "string", 
+            "description": "Bounding box as maximum values for S W N E. Example: 12.6 -119.4 19.9 84.8."
+
+        },
+        "layer_id_s": { 
+            "type": "string", 
+            "description": "The complete identifier for the layer via WMS/WFS/WCS protocol. Example: druid:vr593vj7147."
+
+        },
+        "layer_geom_type_s": { 
+            "type": "string",
+            "enum": ["Point", "Line", "Polygon", "Raster", "Scanned Map", "Mixed"],
+            "description": "Geometry type for layer data, using controlled vocabulary."
+        },
+        "layer_modified_dt": { 
+            "type": "string", 
+            "format": "date-time",
+            "description": "Last modification date for the metadata record, using XML Schema dateTime format (YYYY-MM-DDThh:mm:ssZ)."
+        },
+        "layer_slug_s": { 
+            "type": "string", 
+            "description": "Unique identifier visible to the user, used for Permalinks. Example: stanford-vr593vj7147."
+
+        },
+        "solr_geom": { 
+            "type": "string",
+            "pattern": "ENVELOPE(.*,.*,.*,.*)",
+            "description": "Derived from georss_polygon_s or georss_box_s. Shape of the layer as a ENVELOPE WKT using W E N S. Example: ENVELOPE(76.76, 84.76, 19.91, 12.62). Note that this field is indexed as a Solr spatial (RPT) field."
+
+        },
+        "solr_year_i": { 
+            "type": "integer",
+            "description": "Derived from dct_temporal_sm. Year for which layer is valid and only a single value. Example: 1989. Note that this field is indexed as a Solr numeric field."
+
+        },
+        "dc_creator_sm": { 
+            "type": "array",
+            "items": {
+                 "type": "string"
+             },
+            "description": "Author(s). Example: George Washington, Thomas Jefferson."
+
+        },
+        "dc_format_s": { 
+            "type": "string", 
+            "enum": ["Shapefile", "GeoTIFF", "ArcGRID"],
+            "description": "File format for the layer, using a controlled vocabulary."
+
+        },
+        "dc_language_s": { 
+            "type": "string", 
+            "description": "Language for the layer. Example: English."
+
+        },
+        "dc_publisher_s": { 
+            "type": "string", 
+            "description": "Publisher. Example: ML InfoMap."
+
+        },
+        "dc_subject_sm": { 
+            "type": "array",
+            "items": {
+                 "type": "string"
+             },
+            "description": "Subjects, preferrably in a controlled vocabulary. Examples: Census, Human settlements."
+
+        },
+        "dc_type_s": { 
+            "type": "string",
+            "enum": ["Dataset", "Image", "PhysicalObject"],
+            "description": "Resource type, using DCMI Type Vocabulary."
+
+        },
+        "dct_spatial_sm": { 
+            "type": "array",
+            "items": {
+                 "type": "string"
+             },
+            "description": "Spatial coverage and place names, preferrably in a controlled vocabulary. Example: 'Paris, France'."
+
+        },
+        "dct_temporal_sm": { 
+            "type": "array", 
+            "items": {
+                 "type": "string"
+             },
+            "description": "Temporal coverage, typically years or dates. Example: 1989, circa 2010, 2007-2009. Note that this field is not in a specific date format."
+
+        },
+        "dct_issued_dt": { 
+            "type": "string",
+            "format": "date-time",
+            "description": "Issued date for the layer, using XML Schema dateTime format (YYYY-MM-DDThh:mm:ssZ)."
+
+        },
+        "dct_isPartOf_sm": { 
+            "type": "array",
+            "items": {
+                 "type": "string"
+             },
+            "description": "Holding dataset for the layer, such as the name of a collection. Example: Village Maps of India."
+
+        },
+        "georss_point_s": { 
+            "type": "string", 
+            "description": "Point representation for layer as y, x - i.e., centroid. Example: 12.6 -119.4."
+
+        }
+    }
+}

data/lib/tasks/geo_combine.rake

@@ -4,32 +4,35 @@ require 'json'
 require 'rsolr'
 
 namespace :geocombine do
+  ogm_path = ENV['OGM_PATH'] || 'tmp/opengeometadata'
+  solr_url = ENV['SOLR_URL'] || 'http://127.0.0.1:8983/solr/blacklight-core'
   desc 'Clone all OpenGeoMetadata repositories'
   task :clone do
     ogm_api_uri = URI('https://api.github.com/orgs/opengeometadata/repos')
     ogm_repos = JSON.parse(Net::HTTP.get(ogm_api_uri)).map{ |repo| repo['git_url']}
     ogm_repos.each do |repo|
-      if repo =~ /^git:\/\/github.com\/OpenGeoMetadata\/edu.*/
-        system "cd tmp && git clone #{repo}"
+      if repo =~ /^git:\/\/github.com\/OpenGeoMetadata\/(edu|org|uk)\..*/
+        system "mkdir -p #{ogm_path} && cd #{ogm_path} && git clone #{repo}"
       end
     end
   end
   desc '"git pull" OpenGeoMetadata repositories'
   task :pull do
-    Dir.glob('tmp/*').map{ |dir| system "cd #{dir} && git pull origin master" if dir =~ /.*edu.*./ }
+    Dir.glob("#{ogm_path}/*").map{ |dir| system "cd #{dir} && git pull origin master" if dir =~ /.*(edu|org|uk)\..*./ }
   end
   desc 'Index all of the GeoBlacklight documents'
   task :index do
-    solr = RSolr.connect :url => 'http://127.0.0.1:8983/solr'
-    Find.find('tmp') do |path|
-      if path =~ /.*geoblacklight.xml$/
-        doc = File.read(path)
-        begin
-          solr.update data: doc
-          solr.commit
-        rescue RSolr::Error::Http => error
-          puts error
-        end
+    solr = RSolr.connect :url => solr_url
+    Find.find(ogm_path) do |path|
+      next unless path =~ /.*geoblacklight.json$/
+      doc = JSON.parse(File.read(path))
+      begin
+        solr.update params: { commitWithin: 500, overwrite: true },
+                    data: [doc].to_json,
+                    headers: { 'Content-Type' => 'application/json' }
+
+      rescue RSolr::Error::Http => error
+        puts error
       end
     end
   end