geo_combine 0.4.0 → 0.6.0

data/lib/geo_combine/geo_blacklight_harvester.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+module GeoCombine
+  ##
+  # A class to harvest and index results from GeoBlacklight sites
+  # You can configure the sites to be harvested via a configure command.
+  # GeoCombine::GeoBlacklightHarvester.configure do
+  #   {
+  #     SITE: { host: 'https://example.com', params: { f: { dct_provenance_s: ['SITE'] } } }
+  #   }
+  # end
+  # The class configuration also allows for various other things to be configured:
+  #  - A debug parameter to print out details of what is being harvested and indexed
+  #  - crawl delays for each page of results (globally or on a per site basis)
+  #  - Solr's commitWithin parameter (defaults to 5000)
+  #  - A document transformer proc to modify a document before indexing (defaults to removing _version_, score, and timestamp)
+  # Example: GeoCombine::GeoBlacklightHarvester.new('SITE').index
+  class GeoBlacklightHarvester
+    require 'active_support/core_ext/object/to_query'
+
+    class << self
+      attr_writer :document_transformer
+
+      def configure(&block)
+        @config = yield block
+      end
+
+      def config
+        @config || {}
+      end
+
+      def document_transformer
+        @document_transformer || lambda do |document|
+          document.delete('_version_')
+          document.delete('score')
+          document.delete('timestamp')
+          document.delete('solr_bboxtype__minX')
+          document.delete('solr_bboxtype__minY')
+          document.delete('solr_bboxtype__maxX')
+          document.delete('solr_bboxtype__maxY')
+          document
+        end
+      end
+    end
+
+    attr_reader :site, :site_key
+
+    def initialize(site_key)
+      @site_key = site_key
+      @site = self.class.config[site_key]
+
+      raise ArgumentError, "Site key #{@site_key.inspect} is not configured for #{self.class.name}" unless @site
+    end
+
+    def index
+      puts "Fetching page 1 @ #{base_url}&page=1" if self.class.config[:debug]
+      response = JSON.parse(Net::HTTP.get(URI("#{base_url}&page=1")))
+      response_class = BlacklightResponseVersionFactory.call(response)
+
+      response_class.new(response: response, base_url: base_url).documents.each do |docs|
+        docs.map! do |document|
+          self.class.document_transformer&.call(document)
+        end.compact
+
+        puts "Adding #{docs.count} documents to solr" if self.class.config[:debug]
+        solr_connection.update params: { commitWithin: commit_within, overwrite: true },
+                               data: docs.to_json,
+                               headers: { 'Content-Type' => 'application/json' }
+
+        sleep(crawl_delay.to_i) if crawl_delay
+      end
+    end
+
+    ##
+    # A "factory" class to determine the blacklight response version to use
+    class BlacklightResponseVersionFactory
+      def self.call(json)
+        keys = json.keys
+        if keys.include?('response')
+          LegacyBlacklightResponse
+        elsif keys.any? && %w[links data].all? { |param| keys.include?(param) }
+          ModernBlacklightResponse
+        else
+          raise NotImplementedError,
+                "The following json response was not able to be parsed by the GeoBlacklightHarvester\n#{json}"
+        end
+      end
+    end
+
+    class LegacyBlacklightResponse
+      attr_reader :base_url
+      attr_accessor :response, :page
+
+      def initialize(response:, base_url:)
+        @base_url = base_url
+        @response = response
+        @page = 1
+      end
+
+      def documents
+        return enum_for(:documents) unless block_given?
+
+        while current_page && total_pages && (current_page <= total_pages)
+          yield response.dig('response', 'docs')
+
+          break if current_page == total_pages
+
+          self.page += 1
+          puts "Fetching page #{page} @ #{url}" if GeoCombine::GeoBlacklightHarvester.config[:debug]
+
+          begin
+            self.response = JSON.parse(Net::HTTP.get(URI(url)))
+          rescue StandardError => e
+            puts "Request for #{url} failed with #{e}"
+            self.response = nil
+          end
+        end
+      end
+
+      private
+
+      def url
+        "#{base_url}&page=#{page}"
+      end
+
+      def current_page
+        response.dig('response', 'pages', 'current_page')
+      end
+
+      def total_pages
+        response.dig('response', 'pages', 'total_pages')
+      end
+    end
+
+    ##
+    # Class to return documents from the Blacklight API (v7 and above)
+    class ModernBlacklightResponse
+      attr_reader :base_url
+      attr_accessor :response, :page
+
+      def initialize(response:, base_url:)
+        @base_url = base_url
+        @response = response
+        @page = 1
+      end
+
+      def documents
+        return enum_for(:documents) unless block_given?
+
+        while response && response['data'].any?
+          document_urls = response['data'].collect { |data| data.dig('links', 'self') }.compact
+
+          yield documents_from_urls(document_urls)
+
+          url = response.dig('links', 'next')
+          break unless url
+
+          url = "#{url}&format=json"
+          self.page += 1
+          puts "Fetching page #{page} @ #{url}" if GeoCombine::GeoBlacklightHarvester.config[:debug]
+          begin
+            self.response = JSON.parse(Net::HTTP.get(URI(url)))
+          rescue StandardError => e
+            puts "Request for #{url} failed with #{e}"
+            self.response = nil
+          end
+        end
+      end
+
+      private
+
+      def documents_from_urls(urls)
+        puts "Fetching #{urls.count} documents for page #{page}" if GeoCombine::GeoBlacklightHarvester.config[:debug]
+        urls.map do |url|
+          JSON.parse(Net::HTTP.get(URI("#{url}/raw")))
+        rescue StandardError => e
+          puts "Fetching \"#{url}/raw\" failed with #{e}"
+
+          nil
+        end.compact
+      end
+    end
+
+    private
+
+    def base_url
+      "#{site[:host]}?#{default_params.to_query}"
+    end
+
+    def solr_connection
+      solr_url = ENV['SOLR_URL'] || 'http://127.0.0.1:8983/solr/blacklight-core'
+
+      RSolr.connect url: solr_url, adapter: :net_http_persistent
+    end
+
+    def commit_within
+      self.class.config[:commit_within] || '5000'
+    end
+
+    def crawl_delay
+      site[:crawl_delay] || self.class.config[:crawl_delay]
+    end
+
+    def default_params
+      {
+        per_page: 100,
+        format: :json
+      }.merge(site[:params])
+    end
+  end
+end

data/lib/geo_combine/geoblacklight.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'active_support/core_ext/object/blank'
 require 'active_support/core_ext/hash/except'
 require 'open-uri'
@@ -10,8 +12,8 @@ module GeoCombine
 
     attr_reader :metadata
 
-    GEOBLACKLIGHT_VERSION = 'v1.1.0'
-    SCHEMA_JSON_URL = "https://raw.githubusercontent.com/geoblacklight/geoblacklight/#{GEOBLACKLIGHT_VERSION}/schema/geoblacklight-schema.json".freeze
+    GEOBLACKLIGHT_VERSION = '1.0'
+    SCHEMA_JSON_URL = "https://raw.githubusercontent.com/geoblacklight/geoblacklight/main/schema/geoblacklight-schema-#{GEOBLACKLIGHT_VERSION}.json"
     DEPRECATED_KEYS_V1 = %w[
       uuid
       georss_polygon_s
@@ -29,7 +31,6 @@ module GeoCombine
     # @param [Hash] fields enhancements to metadata that are merged with @metadata
     def initialize(metadata, fields = {})
       @metadata = JSON.parse(metadata).merge(fields)
-      @schema = nil
     end
 
     ##
@@ -58,8 +59,7 @@ module GeoCombine
     # Validates a GeoBlacklight-Schema json document
     # @return [Boolean]
     def valid?
-      @schema ||= JSON.parse(open(SCHEMA_JSON_URL).read)
-      JSON::Validator.validate!(@schema, to_json, fragment: '#/properties/layer') &&
+      JSON::Validator.validate!(schema, to_json, fragment: '#/definitions/layer') &&
         dct_references_validate! &&
         spatial_validate!
     end
@@ -69,9 +69,14 @@ module GeoCombine
     # @return [Boolean]
     def dct_references_validate!
       return true unless metadata.key?('dct_references_s') # TODO: shouldn't we require this field?
+
       begin
         ref = JSON.parse(metadata['dct_references_s'])
-        raise GeoCombine::Exceptions::InvalidDCTReferences, 'dct_references must be parsed to a Hash' unless ref.is_a?(Hash)
+        unless ref.is_a?(Hash)
+          raise GeoCombine::Exceptions::InvalidDCTReferences,
+                'dct_references must be parsed to a Hash'
+        end
+
         true
       rescue JSON::ParserError => e
         raise e, "Invalid JSON in dct_references_s: #{e.message}"
@@ -89,6 +94,7 @@ module GeoCombine
     # GeoBlacklight-Schema format
     def translate_formats(key, value)
       return unless key == 'dc_format_s' && formats.include?(value)
+
       metadata[key] = formats[value]
     end
 
@@ -96,6 +102,7 @@ module GeoCombine
     # Enhances the 'layer_geom_type_s' field by translating from known types
     def translate_geometry_type(key, value)
       return unless key == 'layer_geom_type_s' && geometry_types.include?(value)
+
       metadata[key] = geometry_types[value]
     end
 
@@ -104,6 +111,7 @@ module GeoCombine
     # categories
     def enhance_subjects(key, value)
       return unless key == 'dc_subject_sm'
+
       metadata[key] = value.map do |val|
         if subjects.include?(val)
           subjects[val]
@@ -118,11 +126,13 @@ module GeoCombine
     # and ISO8601 (for indexing into Solr)
     def format_proper_date(key, value)
       return unless key == 'layer_modified_dt'
+
       metadata[key] = Time.parse(value).utc.iso8601
     end
 
     def fields_should_be_array(key, value)
       return unless should_be_array.include?(key) && !value.is_a?(Array)
+
       metadata[key] = [value]
     end
 
@@ -155,5 +165,9 @@ module GeoCombine
       # ensure we have a proper v1 record
       valid?
     end
+
+    def schema
+      @schema ||= JSON.parse(URI.open(SCHEMA_JSON_URL).read)
+    end
   end
 end

data/lib/geo_combine/geometry_types.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   module GeometryTypes
     def geometry_types

data/lib/geo_combine/iso19139.rb

@@ -1,6 +1,7 @@
+# frozen_string_literal: true
+
 module GeoCombine
   class Iso19139 < Metadata
-
     ##
     # Returns a Nokogiri::XSLT object containing the ISO19139 to GeoBlacklight
     # XSL

data/lib/geo_combine/ogp.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'active_support/core_ext/object/blank'
 require 'cgi'
 
@@ -77,15 +79,13 @@ module GeoCombine
     end
 
     def date
-      begin
-        DateTime.rfc3339(metadata['ContentDate'])
-      rescue
-        nil
-      end
+      DateTime.rfc3339(metadata['ContentDate'])
+    rescue StandardError
+      nil
     end
 
     def year
-      date.year unless date.nil?
+      date&.year
     end
 
     ##
@@ -104,9 +104,9 @@ module GeoCombine
     def ogp_formats
       case metadata['DataType']
       when 'Paper Map', 'Raster'
-        return 'GeoTIFF'
+        'GeoTIFF'
       when 'Polygon', 'Point', 'Line'
-        return 'Shapefile'
+        'Shapefile'
       else
         raise ArgumentError, metadata['DataType']
       end
@@ -128,6 +128,7 @@ module GeoCombine
                                  north >= -90 && north <= 90 &&
                                  south >= -90 && south <= 90 &&
                                  west <= east && south <= north
+
       "ENVELOPE(#{west}, #{east}, #{north}, #{south})"
     end
 
@@ -165,6 +166,7 @@ module GeoCombine
 
     def download_uri
       return 'http://schema.org/DownloadAction' if institution == 'Harvard'
+
       'http://schema.org/downloadUrl'
     end
 
@@ -205,7 +207,7 @@ module GeoCombine
       sluggify(filter_name(name))
     end
 
-    SLUG_BLACKLIST = %w[
+    SLUG_STRIP_VALUES = %w[
       SDE_DATA.
       SDE.
       SDE2.
@@ -216,8 +218,8 @@ module GeoCombine
 
     def filter_name(name)
       # strip out schema and usernames
-      SLUG_BLACKLIST.each do |blacklisted|
-        name.sub!(blacklisted, '')
+      SLUG_STRIP_VALUES.each do |strip_val|
+        name.sub!(strip_val, '')
       end
       unless name.size > 1
         # use first word of title is empty name

data/lib/geo_combine/railtie.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   class Railtie < Rails::Railtie
     rake_tasks do

data/lib/geo_combine/subjects.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   ##
   # Translation dictionary to ISO topics

data/lib/geo_combine/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
-  VERSION = '0.4.0'
+  VERSION = '0.6.0'
 end

data/lib/geo_combine.rb

@@ -1,10 +1,11 @@
+# frozen_string_literal: true
+
 require 'nokogiri'
 require 'json'
 require 'json-schema'
 require 'sanitize'
 
 module GeoCombine
-
   ##
   # TODO: Create a parse method that can interpret the type of metadata being
   # passed in.
@@ -23,7 +24,7 @@ module GeoCombine
     # @param [String] metadata can be a File path
     # "./tmp/edu.stanford.purl/bb/338/jh/0716/iso19139.xml" or a String of XML
     # metadata
-    def initialize metadata
+    def initialize(metadata)
       metadata = File.read metadata if File.readable? metadata
       metadata = Nokogiri::XML(metadata) if metadata.instance_of? String
       @metadata = metadata
@@ -35,7 +36,7 @@ module GeoCombine
     # GeoCombine::Geoblacklight on its instantiation
     # @return [GeoCombine::Geoblacklight] the data transformed into
     # geoblacklight schema, returned as a GeoCombine::Geoblacklight
-    def to_geoblacklight fields = {}
+    def to_geoblacklight(fields = {})
       GeoCombine::Geoblacklight.new(xsl_geoblacklight.apply_to(@metadata), fields)
     end
 
@@ -68,6 +69,9 @@ require 'geo_combine/esri_open_data'
 require 'geo_combine/ckan_metadata'
 require 'geo_combine/ogp'
 
+# Require harvesting/indexing files
+require 'geo_combine/geo_blacklight_harvester'
+
 # Require gem files
 require 'geo_combine/version'
 require 'geo_combine/railtie' if defined?(Rails)

data/lib/tasks/geo_combine.rake

@@ -1,16 +1,12 @@
+# frozen_string_literal: true
+
 require 'net/http'
 require 'json'
 require 'rsolr'
 require 'find'
+require 'geo_combine/geo_blacklight_harvester'
 
 namespace :geocombine do
-  commit_within = (ENV['SOLR_COMMIT_WITHIN'] || 5000).to_i
-  ogm_path = ENV['OGM_PATH'] || 'tmp/opengeometadata'
-  solr_url = ENV['SOLR_URL'] || 'http://127.0.0.1:8983/solr/blacklight-core'
-  whitelist = %w[
-    https://github.com/OpenGeoMetadata/big-ten.git
-  ]
-
   desc 'Clone OpenGeoMetadata repositories'
   task :clone, [:repo] do |_t, args|
     if args.repo
@@ -18,46 +14,81 @@ namespace :geocombine do
     else
       ogm_api_uri = URI('https://api.github.com/orgs/opengeometadata/repos')
       ogm_repos = JSON.parse(Net::HTTP.get(ogm_api_uri)).map do |repo|
-        repo['clone_url'] if repo['size'] > 0
+        repo['clone_url'] if (repo['size']).positive?
       end.compact
-      ogm_repos.select! { |repo| whitelist.include?(repo) || repo =~ /(edu|org|uk)\..*\.git$/ }
+      ogm_repos.reject! { |repo| GeoCombineRake.denylist.include?(repo) }
     end
     ogm_repos.each do |repo|
-      system "echo #{repo} && mkdir -p #{ogm_path} && cd #{ogm_path} && git clone --depth 1 #{repo}"
+      Kernel.system "echo #{repo} && mkdir -p #{GeoCombineRake.ogm_path} && cd #{GeoCombineRake.ogm_path} && git clone --depth 1 #{repo}"
     end
   end
 
   desc '"git pull" OpenGeoMetadata repositories'
   task :pull, [:repo] do |_t, args|
     paths = if args.repo
-              [File.join(ogm_path, args.repo)]
+              [File.join(GeoCombineRake.ogm_path, args.repo)]
             else
-              Dir.glob("#{ogm_path}/*")
+              Dir.glob("#{GeoCombineRake.ogm_path}/*")
             end
     paths.each do |path|
       next unless File.directory?(path)
-      system "echo #{path} && cd #{path} && git pull origin"
+
+      Kernel.system "echo #{path} && cd #{path} && git pull origin"
     end
   end
 
-  desc 'Index all of the GeoBlacklight JSON documents'
+  desc 'Index all JSON documents except Layers.json'
   task :index do
-    puts "Indexing #{ogm_path} into #{solr_url}"
-    solr = RSolr.connect url: solr_url, adapter: :net_http_persistent
-    Find.find(ogm_path) do |path|
-      next unless File.basename(path) == 'geoblacklight.json'
+    puts "Indexing #{GeoCombineRake.ogm_path} into #{GeoCombineRake.solr_url}"
+    solr = RSolr.connect url: GeoCombineRake.solr_url, adapter: :net_http_persistent
+    Find.find(GeoCombineRake.ogm_path) do |path|
+      next unless File.basename(path).include?('.json') && File.basename(path) != 'layers.json'
+
       doc = JSON.parse(File.read(path))
       [doc].flatten.each do |record|
-        begin
-          puts "Indexing #{record['layer_slug_s']}: #{path}" if $DEBUG
-          solr.update params: { commitWithin: commit_within, overwrite: true },
-                      data: [record].to_json,
-                      headers: { 'Content-Type' => 'application/json' }
-        rescue RSolr::Error::Http => error
-          puts error
-        end
+        puts "Indexing #{record['layer_slug_s']}: #{path}" if $DEBUG
+        solr.update params: { commitWithin: GeoCombineRake.commit_within, overwrite: true },
+                    data: [record].to_json,
+                    headers: { 'Content-Type' => 'application/json' }
+      rescue RSolr::Error::Http => e
+        puts e
       end
     end
     solr.commit
   end
+
+  namespace :geoblacklight_harvester do
+    desc 'Harvest documents from a configured GeoBlacklight instance'
+    task :index, [:site] => [:environment] do |_t, args|
+      raise ArgumentError, 'A site argument is required' unless args.site
+
+      GeoCombine::GeoBlacklightHarvester.new(args.site.to_sym).index
+    end
+  end
+end
+
+# Class to hold helper methods for use in GeoCombine rake tasks
+class GeoCombineRake
+  def self.commit_within
+    (ENV['SOLR_COMMIT_WITHIN'] || 5000).to_i
+  end
+
+  def self.denylist
+    [
+      'https://github.com/OpenGeoMetadata/GeoCombine.git',
+      'https://github.com/OpenGeoMetadata/aardvark.git',
+      'https://github.com/OpenGeoMetadata/metadata-issues.git',
+      'https://github.com/OpenGeoMetadata/ogm_utils-python.git',
+      'https://github.com/OpenGeoMetadata/opengeometadata.github.io.git',
+      'https://github.com/OpenGeoMetadata/opengeometadata-rails.git'
+    ]
+  end
+
+  def self.ogm_path
+    ENV['OGM_PATH'] || 'tmp/opengeometadata'
+  end
+
+  def self.solr_url
+    ENV['SOLR_URL'] || 'http://127.0.0.1:8983/solr/blacklight-core'
+  end
 end