geo_combine 0.1.0 → 0.5.0

data/lib/geo_combine/esri_open_data.rb

@@ -26,14 +26,12 @@ module GeoCombine
     # @return [Hash]
     def geoblacklight_terms
       {
-        uuid: @metadata['id'],
         dc_identifier_s: @metadata['id'],
         dc_title_s: @metadata['name'],
         dc_description_s: sanitize_and_remove_lines(@metadata['description']),
         dc_rights_s: 'Public',
         dct_provenance_s: @metadata['owner'],
         dct_references_s: references,
-        georss_box_s: georss_box,
         # layer_id_s is used for describing a layer id for a web serivce (WMS, WFS) but is still a required field
         layer_id_s: '',
         layer_geom_type_s: @metadata['geometry_type'],
@@ -62,13 +60,6 @@ module GeoCombine
       }
     end
 
-    ##
-    # Builds a GeoRSS box
-    # @return [String]
-    def georss_box
-      "#{south} #{west} #{north} #{east}"
-    end
-
     ##
     # Builds a Solr Envelope using CQL syntax
     # @return [String]

data/lib/geo_combine/exceptions.rb

@@ -0,0 +1,8 @@
+module GeoCombine
+  module Exceptions
+    class InvalidDCTReferences < StandardError
+    end
+    class InvalidGeometry < StandardError
+    end
+  end
+end

data/lib/geo_combine/formatting.rb

@@ -15,7 +15,7 @@ module GeoCombine
     # @param [String] text
     # @return [String]
     def remove_lines(text)
-      text.gsub(/\n/, '')
+      text.delete("\n")
     end
 
     ##
@@ -25,5 +25,10 @@ module GeoCombine
     def sanitize_and_remove_lines(text)
       remove_lines(sanitize(text))
     end
+
+    # slugs should be lowercase and only have a-z, A-Z, 0-9, and -
+    def sluggify(slug)
+      slug.gsub(/[^a-zA-Z0-9\-]/, '-').gsub(/[\-]+/, '-').downcase
+    end
   end
 end

data/lib/geo_combine/geo_blacklight_harvester.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+module GeoCombine
+  ##
+  # A class to harvest and index results from GeoBlacklight sites
+  # You can configure the sites to be harvested via a configure command.
+  # GeoCombine::GeoBlacklightHarvester.configure do
+  #   {
+  #     SITE: { host: 'https://example.com', params: { f: { dct_provenance_s: ['SITE'] } } }
+  #   }
+  # end
+  # The class configuration also allows for various other things to be configured:
+  #  - A debug parameter to print out details of what is being harvested and indexed
+  #  - crawl delays for each page of results (globally or on a per site basis)
+  #  - Solr's commitWithin parameter (defaults to 5000)
+  #  - A document transformer proc to modify a document before indexing (defaults to removing _version_, score, and timestamp)
+  # Example: GeoCombine::GeoBlacklightHarvester.new('SITE').index
+  class GeoBlacklightHarvester
+    require 'active_support/core_ext/object/to_query'
+
+    class << self
+      attr_writer :document_transformer
+
+      def configure(&block)
+        @config = yield block
+      end
+
+      def config
+        @config || {}
+      end
+
+      def document_transformer
+        @document_transformer || ->(document) do
+          document.delete('_version_')
+          document.delete('score')
+          document.delete('timestamp')
+          document
+        end
+      end
+    end
+
+
+    attr_reader :site, :site_key
+    def initialize(site_key)
+      @site_key = site_key
+      @site = self.class.config[site_key]
+
+      raise ArgumentError, "Site key #{@site_key.inspect} is not configured for #{self.class.name}" unless @site
+    end
+
+    def index
+      puts "Fetching page 1 @ #{base_url}&page=1" if self.class.config[:debug]
+      response = JSON.parse(Net::HTTP.get(URI("#{base_url}&page=1")))
+      response_class = BlacklightResponseVersionFactory.call(response)
+
+      response_class.new(response: response, base_url: base_url).documents.each do |docs|
+        docs.map! do |document|
+          self.class.document_transformer.call(document) if self.class.document_transformer
+        end.compact
+
+        puts "Adding #{docs.count} documents to solr" if self.class.config[:debug]
+        solr_connection.update params: { commitWithin: commit_within, overwrite: true },
+                               data: docs.to_json,
+                               headers: { 'Content-Type' => 'application/json' }
+
+        sleep(crawl_delay.to_i) if crawl_delay
+      end
+    end
+
+    ##
+    # A "factory" class to determine the blacklight response version to use
+    class BlacklightResponseVersionFactory
+      def self.call(json)
+        keys = json.keys
+        if keys.include?('response')
+          LegacyBlacklightResponse
+        elsif keys.any? && %w[links data].all? { |param| keys.include?(param) }
+          ModernBlacklightResponse
+        else
+          raise NotImplementedError, "The following json response was not able to be parsed by the GeoBlacklightHarvester\n#{json}"
+        end
+      end
+    end
+
+    class LegacyBlacklightResponse
+      attr_reader :base_url
+      attr_accessor :response, :page
+      def initialize(response:, base_url:)
+        @base_url = base_url
+        @response = response
+        @page = 1
+      end
+
+      def documents
+        return enum_for(:documents) unless block_given?
+
+        while current_page && total_pages && (current_page <= total_pages) do
+          yield response.dig('response', 'docs')
+
+          break if current_page == total_pages
+          self.page += 1
+          puts "Fetching page #{page} @ #{url}" if GeoCombine::GeoBlacklightHarvester.config[:debug]
+
+          begin
+            self.response = JSON.parse(Net::HTTP.get(URI(url)))
+          rescue => e
+            puts "Request for #{url} failed with #{e}"
+            self.response = nil
+          end
+        end
+      end
+
+      private
+
+      def url
+        "#{base_url}&page=#{page}"
+      end
+
+      def current_page
+        response.dig('response', 'pages', 'current_page')
+      end
+
+      def total_pages
+        response.dig('response', 'pages', 'total_pages')
+      end
+    end
+
+    ##
+    # Class to return documents from the Blacklight API (v7 and above)
+    class ModernBlacklightResponse
+      attr_reader :base_url
+      attr_accessor :response, :page
+      def initialize(response:, base_url:)
+        @base_url = base_url
+        @response = response
+        @page = 1
+      end
+
+      def documents
+        return enum_for(:documents) unless block_given?
+
+        while response && response['data'].any?
+          document_urls = response['data'].collect { |data| data.dig('links', 'self') }.compact
+
+          yield documents_from_urls(document_urls)
+
+          url = response.dig('links', 'next')
+          break unless url
+          self.page += 1
+          puts "Fetching page #{page} @ #{url}" if GeoCombine::GeoBlacklightHarvester.config[:debug]
+          begin
+            self.response = JSON.parse(Net::HTTP.get(URI(url)))
+          rescue => e
+            puts "Request for #{url} failed with #{e}"
+            self.response = nil
+          end
+        end
+      end
+
+      private
+
+      def documents_from_urls(urls)
+        puts "Fetching #{urls.count} documents for page #{page}" if GeoCombine::GeoBlacklightHarvester.config[:debug]
+        urls.map do |url|
+          begin
+            JSON.parse(Net::HTTP.get(URI("#{url}/raw")))
+          rescue => e
+            puts "Fetching \"#{url}/raw\" failed with #{e}"
+
+            nil
+          end
+        end.compact
+      end
+    end
+
+    private
+
+    def base_url
+      "#{site[:host]}?#{default_params.to_query}"
+    end
+
+    def solr_connection
+      solr_url = ENV['SOLR_URL'] || 'http://127.0.0.1:8983/solr/blacklight-core'
+
+      RSolr.connect url: solr_url, adapter: :net_http_persistent
+    end
+
+    def commit_within
+      self.class.config[:commit_within] || '5000'
+    end
+
+    def crawl_delay
+      site[:crawl_delay] || self.class.config[:crawl_delay]
+    end
+
+    def default_params
+      {
+        per_page: 100,
+        format: :json
+      }.merge(site[:params])
+    end
+  end
+end

data/lib/geo_combine/geoblacklight.rb

@@ -1,3 +1,7 @@
+require 'active_support/core_ext/object/blank'
+require 'active_support/core_ext/hash/except'
+require 'open-uri'
+
 module GeoCombine
   class Geoblacklight
     include GeoCombine::Formats
@@ -6,6 +10,18 @@ module GeoCombine
 
     attr_reader :metadata
 
+    GEOBLACKLIGHT_VERSION = 'v1.1.0'
+    SCHEMA_JSON_URL = "https://raw.githubusercontent.com/geoblacklight/geoblacklight/#{GEOBLACKLIGHT_VERSION}/schema/geoblacklight-schema.json".freeze
+    DEPRECATED_KEYS_V1 = %w[
+      uuid
+      georss_polygon_s
+      georss_point_s
+      georss_box_s
+      dc_relation_sm
+      solr_issued_i
+      solr_bbox
+    ].freeze
+
     ##
     # Initializes a GeoBlacklight object
     # @param [String] metadata be a valid JSON string document in
@@ -13,13 +29,16 @@ module GeoCombine
     # @param [Hash] fields enhancements to metadata that are merged with @metadata
     def initialize(metadata, fields = {})
       @metadata = JSON.parse(metadata).merge(fields)
+      @schema = nil
     end
 
     ##
     # Calls metadata enhancement methods for each key, value pair in the
     # metadata hash
     def enhance_metadata
-      @metadata.each do |key, value|
+      upgrade_to_v1 if metadata['geoblacklight_version'].blank?
+
+      metadata.each do |key, value|
         translate_formats(key, value)
         enhance_subjects(key, value)
         format_proper_date(key, value)
@@ -31,16 +50,36 @@ module GeoCombine
     ##
     # Returns a string of JSON from a GeoBlacklight hash
     # @return (String)
-    def to_json
-      @metadata.to_json
+    def to_json(options = {})
+      metadata.to_json(options)
     end
 
     ##
     # Validates a GeoBlacklight-Schema json document
     # @return [Boolean]
     def valid?
-      schema = JSON.parse(File.read(File.join(File.dirname(__FILE__), '../schema/geoblacklight-schema.json')))
-      JSON::Validator.validate!(schema, to_json, validate_schema: true)
+      @schema ||= JSON.parse(open(SCHEMA_JSON_URL).read)
+      JSON::Validator.validate!(@schema, to_json, fragment: '#/properties/layer') &&
+        dct_references_validate! &&
+        spatial_validate!
+    end
+
+    ##
+    # Validate dct_references_s
+    # @return [Boolean]
+    def dct_references_validate!
+      return true unless metadata.key?('dct_references_s') # TODO: shouldn't we require this field?
+      begin
+        ref = JSON.parse(metadata['dct_references_s'])
+        raise GeoCombine::Exceptions::InvalidDCTReferences, 'dct_references must be parsed to a Hash' unless ref.is_a?(Hash)
+        true
+      rescue JSON::ParserError => e
+        raise e, "Invalid JSON in dct_references_s: #{e.message}"
+      end
+    end
+
+    def spatial_validate!
+      GeoCombine::BoundingBox.from_envelope(metadata['solr_geom']).valid?
     end
 
     private
@@ -49,43 +88,72 @@ module GeoCombine
     # Enhances the 'dc_format_s' field by translating a format type to a valid
     # GeoBlacklight-Schema format
     def translate_formats(key, value)
-      @metadata[key] = formats[value] if key == 'dc_format_s' && formats.include?(value)
+      return unless key == 'dc_format_s' && formats.include?(value)
+      metadata[key] = formats[value]
     end
 
     ##
     # Enhances the 'layer_geom_type_s' field by translating from known types
     def translate_geometry_type(key, value)
-      @metadata[key] = geometry_types[value] if key == 'layer_geom_type_s' && geometry_types.include?(value)
+      return unless key == 'layer_geom_type_s' && geometry_types.include?(value)
+      metadata[key] = geometry_types[value]
     end
 
     ##
     # Enhances the 'dc_subject_sm' field by translating subjects to ISO topic
     # categories
     def enhance_subjects(key, value)
-      @metadata[key] = value.map do |val|
+      return unless key == 'dc_subject_sm'
+      metadata[key] = value.map do |val|
         if subjects.include?(val)
           subjects[val]
         else
           val
         end
-      end if key == 'dc_subject_sm'
+      end
     end
 
     ##
     # Formats the 'layer_modified_dt' to a valid valid RFC3339 date/time string
     # and ISO8601 (for indexing into Solr)
     def format_proper_date(key, value)
-      @metadata[key] = Time.parse(value).utc.iso8601 if key == 'layer_modified_dt'
+      return unless key == 'layer_modified_dt'
+      metadata[key] = Time.parse(value).utc.iso8601
     end
 
     def fields_should_be_array(key, value)
-      @metadata[key] = [value] if should_be_array.include?(key) && !value.kind_of?(Array)
+      return unless should_be_array.include?(key) && !value.is_a?(Array)
+      metadata[key] = [value]
     end
 
     ##
     # GeoBlacklight-Schema fields that should be type Array
     def should_be_array
-      ['dc_creator_sm', 'dc_subject_sm', 'dct_spatial_sm', 'dct_temporal_sm', 'dct_isPartOf_sm']
+      %w[
+        dc_creator_sm
+        dc_subject_sm
+        dct_spatial_sm
+        dct_temporal_sm
+        dct_isPartOf_sm
+      ].freeze
+    end
+
+    ##
+    # Converts a pre-v1.0 schema into a compliant v1.0 schema
+    def upgrade_to_v1
+      metadata['geoblacklight_version'] = '1.0'
+
+      # ensure required fields
+      metadata['dc_identifier_s'] = metadata['uuid'] if metadata['dc_identifier_s'].blank?
+
+      # normalize to alphanum and - only
+      metadata['layer_slug_s'].gsub!(/[^[[:alnum:]]]+/, '-') if metadata['layer_slug_s'].present?
+
+      # remove deprecated fields
+      metadata.except!(*DEPRECATED_KEYS_V1)
+
+      # ensure we have a proper v1 record
+      valid?
     end
   end
 end

data/lib/geo_combine/ogp.rb

@@ -0,0 +1,229 @@
+require 'active_support/core_ext/object/blank'
+require 'cgi'
+
+module GeoCombine
+  # Data model for OpenGeoPortal metadata
+  class OGP
+    class InvalidMetadata < RuntimeError; end
+    include GeoCombine::Formatting
+    attr_reader :metadata
+
+    ##
+    # Initializes an OGP object for parsing
+    # @param [String] metadata a valid serialized JSON string from OGP instance
+    # @raise [InvalidMetadata]
+    def initialize(metadata)
+      @metadata = JSON.parse(metadata)
+      raise InvalidMetadata unless valid?
+    end
+
+    OGP_REQUIRED_FIELDS = %w[
+      Access
+      Institution
+      LayerDisplayName
+      LayerId
+      MaxX
+      MaxY
+      MinX
+      MinY
+      Name
+    ].freeze
+
+    ##
+    # Runs validity checks on OGP metadata to ensure fields are present
+    def valid?
+      OGP_REQUIRED_FIELDS.all? { |k| metadata[k].present? }
+    end
+
+    ##
+    # Creates and returns a Geoblacklight schema object from this metadata
+    # @return [GeoCombine::Geoblacklight]
+    def to_geoblacklight
+      GeoCombine::Geoblacklight.new(geoblacklight_terms.to_json)
+    end
+
+    ##
+    # Builds a Geoblacklight Schema type hash from Esri Open Data portal
+    # metadata
+    # @return [Hash]
+    def geoblacklight_terms
+      {
+        # Required fields
+        dc_identifier_s: identifier,
+        layer_slug_s: slug,
+        dc_title_s: metadata['LayerDisplayName'],
+        solr_geom: envelope,
+        dct_provenance_s: institution,
+        dc_rights_s: metadata['Access'],
+        geoblacklight_version: '1.0',
+
+        # Recommended fields
+        dc_description_s: metadata['Abstract'],
+        layer_geom_type_s: ogp_geom,
+        dct_references_s: references,
+        layer_id_s: "#{metadata['WorkspaceName']}:#{metadata['Name']}",
+
+        # Optional
+        dct_temporal_sm: [metadata['ContentDate']],
+        dc_format_s: ogp_formats,
+        # dct_issued_dt
+        # dc_language_s
+        dct_spatial_sm: placenames,
+        solr_year_i: year,
+        dc_publisher_s: metadata['Publisher'],
+        dc_subject_sm: subjects,
+        dc_type_s: 'Dataset'
+      }.delete_if { |_k, v| v.nil? }
+    end
+
+    def date
+      begin
+        DateTime.rfc3339(metadata['ContentDate'])
+      rescue
+        nil
+      end
+    end
+
+    def year
+      date.year unless date.nil?
+    end
+
+    ##
+    # Convert "Paper Map" to Raster, assumes all OGP "Paper Maps" have WMS
+    def ogp_geom
+      case metadata['DataType']
+      when 'Paper Map'
+        'Raster'
+      else
+        metadata['DataType']
+      end
+    end
+
+    ##
+    # OGP doesn't ship format types, so we just try and be clever here.
+    def ogp_formats
+      case metadata['DataType']
+      when 'Paper Map', 'Raster'
+        return 'GeoTIFF'
+      when 'Polygon', 'Point', 'Line'
+        return 'Shapefile'
+      else
+        raise ArgumentError, metadata['DataType']
+      end
+    end
+
+    ##
+    # Converts references to json
+    # @return [String]
+    def references
+      references_hash.to_json
+    end
+
+    ##
+    # Builds a Solr Envelope using CQL syntax
+    # @return [String]
+    def envelope
+      raise ArgumentError unless west >= -180 && west <= 180 &&
+                                 east >= -180 && east <= 180 &&
+                                 north >= -90 && north <= 90 &&
+                                 south >= -90 && south <= 90 &&
+                                 west <= east && south <= north
+      "ENVELOPE(#{west}, #{east}, #{north}, #{south})"
+    end
+
+    def subjects
+      fgdc.metadata.xpath('//themekey').map(&:text) if fgdc
+    end
+
+    def placenames
+      fgdc.metadata.xpath('//placekey').map(&:text) if fgdc
+    end
+
+    def fgdc
+      GeoCombine::Fgdc.new(metadata['FgdcText']) if metadata['FgdcText']
+    end
+
+    private
+
+    ##
+    # Builds references used for dct_references
+    # @return [Hash]
+    def references_hash
+      results = {
+        'http://www.opengis.net/def/serviceType/ogc/wfs' => location['wfs'],
+        'http://www.opengis.net/def/serviceType/ogc/wms' => location['wms'],
+        'http://schema.org/url' => location['url'],
+        download_uri => location['download']
+      }
+
+      # Handle null, "", and [""]
+      results.map { |k, v| { k => ([] << v).flatten.first } if v }
+             .flatten
+             .compact
+             .reduce({}, :merge)
+    end
+
+    def download_uri
+      return 'http://schema.org/DownloadAction' if institution == 'Harvard'
+      'http://schema.org/downloadUrl'
+    end
+
+    ##
+    # OGP "Location" field parsed
+    def location
+      JSON.parse(metadata['Location'])
+    end
+
+    def north
+      metadata['MaxY'].to_f
+    end
+
+    def south
+      metadata['MinY'].to_f
+    end
+
+    def east
+      metadata['MaxX'].to_f
+    end
+
+    def west
+      metadata['MinX'].to_f
+    end
+
+    def institution
+      metadata['Institution']
+    end
+
+    def identifier
+      CGI.escape(metadata['LayerId']) # TODO: why are we using CGI.escape?
+    end
+
+    def slug
+      name = metadata['LayerId'] || metadata['Name'] || ''
+      name = [institution, name].join('-') if institution.present? &&
+                                              !name.downcase.start_with?(institution.downcase)
+      sluggify(filter_name(name))
+    end
+
+    SLUG_BLACKLIST = %w[
+      SDE_DATA.
+      SDE.
+      SDE2.
+      GISPORTAL.GISOWNER01.
+      GISDATA.
+      MORIS.
+    ].freeze
+
+    def filter_name(name)
+      # strip out schema and usernames
+      SLUG_BLACKLIST.each do |blacklisted|
+        name.sub!(blacklisted, '')
+      end
+      unless name.size > 1
+        # use first word of title is empty name
+        name = metadata['LayerDisplayName'].split.first
+      end
+      name
+    end
+  end
+end