geo_combine 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c91fb8acc12d958eb450bd5cebc525073f30b6df664bd55f80622a942d8e6703
-  data.tar.gz: abcc395544fe5530d2fa9b15d1f26f2e758f5b48983fcab35f8fe7305e424e92
+  metadata.gz: 5b168ca81c2b6d5ff2fa0ce75d18154ad9806f5a872cc6f96e41ab0cea628864
+  data.tar.gz: 9ba88c0cca642ebe79301182f992f2dfe223bc1cf4abaa9b00ac4687293b78f4
 SHA512:
-  metadata.gz: 8ad8cee3e3aa481816dffe885db30217daaa3d5167240ff8395954f02226c8d54a8ce9227efa602127ac55f08106a77f1f1ca3ac0a3203fd51c92fe284327ecd
-  data.tar.gz: 523b156be9684071cd06ddcd6a10dbb14b93f3b35970ad976787edb7363667301c72963075563cccc7e330f4c9a0acad26df271f63a799e1eeaa5dc72373f574
+  metadata.gz: 8c54ead8b591bd20fc3b62fddbfec2b0ac6830972cd85bf49fd31f230c84c8cf1a7bd4c0b56de702a235eca130fc643e4c796cb84adf15fd93f0b73075161fe2
+  data.tar.gz: 28fe9a5209dd77c2f8b60e87fddec8c7892ddf33794afd75522cf09c6827496c445c6c2f52445ad7fa46508ac1a4dcedd0c5068e161d0e176ec7ab232b21d296

data/.github/workflows/ruby.yml

@@ -10,7 +10,7 @@ jobs:
     - name: Set up Ruby and install dependencies
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 2.7
+        ruby-version: 3.1
         bundler-cache: true
     - name: Run linter
       run: bundle exec rubocop
@@ -19,11 +19,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby: [2.7, '3.0', 3.1]
-        faraday_version: [''] # Defaults to whatever's the most recent version.
-        include:
-          - ruby: 2.7
-            faraday_version: '~> 1.0'
+        ruby: [3.1, 3.2, 3.3]
+        faraday_version: ['', '~> 1.0'] # Defaults to whatever's the most recent version.
     steps:
     - uses: actions/checkout@v2
 

data/.rubocop.yml

@@ -5,7 +5,7 @@ require:
 inherit_from: .rubocop_todo.yml
 
 AllCops:
-  TargetRubyVersion: 2.7
+  TargetRubyVersion: 3.1
   DisplayCopNames: true
   NewCops: enable
   Exclude:
@@ -16,6 +16,9 @@ AllCops:
 RSpec/DescribeClass:
   Enabled: false
 
+RSpec/MultipleMemoizedHelpers:
+  Enabled: false
+
 RSpec/BeforeAfterAll:
   Exclude:
     - 'spec/lib/tasks/geo_combine_spec.rb'

data/.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2023-03-27 19:15:05 UTC using RuboCop version 1.48.1.
+# on 2023-09-13 18:53:11 UTC using RuboCop version 1.56.3.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -12,6 +12,7 @@ Lint/RescueException:
     - 'spec/helpers.rb'
 
 # Offense count: 1
+# This cop supports unsafe autocorrection (--autocorrect-all).
 Lint/UselessAssignment:
   Exclude:
     - 'spec/helpers.rb'
@@ -32,17 +33,17 @@ Metrics/BlockLength:
 Metrics/ClassLength:
   Max: 152
 
-# Offense count: 4
+# Offense count: 5
 # Configuration parameters: AllowedMethods, AllowedPatterns.
 Metrics/CyclomaticComplexity:
   Max: 11
 
-# Offense count: 11
+# Offense count: 13
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 Metrics/MethodLength:
   Max: 21
 
-# Offense count: 1
+# Offense count: 2
 # Configuration parameters: AllowedMethods, AllowedPatterns.
 Metrics/PerceivedComplexity:
   Max: 11
@@ -66,13 +67,6 @@ RSpec/ExpectInHook:
     - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
     - 'spec/lib/geo_combine/geoblacklight_spec.rb'
 
-# Offense count: 1
-# Configuration parameters: Include, CustomTransform, IgnoreMethods, SpecSuffixOnly.
-# Include: **/*_spec*rb*, **/spec/**/*
-RSpec/FilePath:
-  Exclude:
-    - 'spec/lib/geo_combine_spec.rb'
-
 # Offense count: 23
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: have_received, receive
@@ -110,16 +104,19 @@ RSpec/OverwritingSetup:
   Exclude:
     - 'spec/lib/geo_combine/geoblacklight_spec.rb'
 
-# Offense count: 1
-RSpec/PendingWithoutReason:
-  Exclude:
-    - 'spec/lib/geo_combine/migrators/v1_aardvark_migrator_spec.rb'
-
 # Offense count: 2
 RSpec/RepeatedExampleGroupBody:
   Exclude:
     - 'spec/lib/geo_combine/iso19139_spec.rb'
 
+# Offense count: 1
+# Configuration parameters: Include, CustomTransform, IgnoreMethods, IgnoreMetadata.
+# Include: **/*_spec.rb
+RSpec/SpecFilePathFormat:
+  Exclude:
+    - '**/spec/routing/**/*'
+    - 'spec/lib/geo_combine_spec.rb'
+
 # Offense count: 19
 RSpec/StubbedMock:
   Exclude:
@@ -145,7 +142,7 @@ Security/Open:
   Exclude:
     - 'lib/geo_combine/geoblacklight.rb'
 
-# Offense count: 7
+# Offense count: 6
 # Configuration parameters: AllowedConstants.
 Style/Documentation:
   Exclude:
@@ -158,9 +155,9 @@ Style/Documentation:
     - 'lib/geo_combine/geometry_types.rb'
     - 'lib/geo_combine/iso19139.rb'
 
-# Offense count: 8
+# Offense count: 12
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns.
 # URISchemes: http, https
 Layout/LineLength:
-  Max: 159
+  Max: 198

data/README.md

@@ -1,12 +1,12 @@
 # GeoCombine
 
-   ![CI](https://github.com/OpenGeoMetadata/GeoCombine/actions/workflows/ruby.yml/badge.svg)
+![CI](https://github.com/OpenGeoMetadata/GeoCombine/actions/workflows/ruby.yml/badge.svg)
 | [![Coverage Status](https://img.shields.io/badge/coverage-95%25-brightgreen)]()
 | [![Gem Version](https://img.shields.io/gem/v/geo_combine.svg)](https://github.com/OpenGeoMetadata/GeoCombine/releases)
 
-
 A Ruby toolkit for managing geospatial metadata, including:
-- tasks for cloning, updating, and indexing OpenGeoMetdata metadata
+
+- tasks for cloning, updating, and indexing OpenGeoMetadata metadata
 - library for converting metadata between standards
 
 ## Installation
@@ -19,11 +19,15 @@ gem 'geo_combine'
 
 And then execute:
 
-    $ bundle install
+```sh
+$ bundle install
+```
 
 Or install it yourself as:
 
-    $ gem install geo_combine
+```sh
+$ gem install geo_combine
+```
 
 ## Usage
 
@@ -43,8 +47,42 @@ Or install it yourself as:
 > iso_metadata.to_html
 ```
 
+### Migrating metadata
+
+You can use the `GeoCombine::Migrators` to migrate metadata from one schema to another.
+
+Currently, the only migrator is `GeoCombine::Migrators::V1AardvarkMigrator` which migrates from the [GeoBlacklight v1 schema](https://github.com/OpenGeoMetadata/opengeometadata.github.io/blob/main/docs/gbl-1.0.md) to the [Aardvark schema](https://github.com/OpenGeoMetadata/opengeometadata.github.io/blob/main/docs/ogm-aardvark.md)
+
+```ruby
+# Load a record in geoblacklight v1 schema
+record = JSON.parse(File.read('.spec/fixtures/docs/full_geoblacklight.json'))
+
+# Migrate it to Aardvark schema
+GeoCombine::Migrators::V1AardvarkMigrator.new(v1_hash: record).run
+```
+
+Some fields cannot be migrated automatically. To handle the migration of collection names to IDs when migrating from v1 to Aardvark, you can provide a mapping of collection names to IDs to the migrator:
+
+```ruby
+# You can store this mapping as a JSON or CSV file and load it into a hash
+id_map = {
+  'My Collection 1' => 'institution:my-collection-1',
+  'My Collection 2' => 'institution:my-collection-2'
+}
+
+GeoCombine::Migrators::V1AardvarkMigrator.new(v1_hash: record, collection_id_map: id_map).run
+```
+
 ### OpenGeoMetadata
 
+#### Logging
+
+Some of the tools and scripts in this gem use Ruby's `Logger` class to print information to `$stderr`. By default, the log level is set to `Logger::INFO`. For more verbose information, you can set the `LOG_LEVEL` environment variable to `DEBUG`:
+
+```sh
+$ LOG_LEVEL=DEBUG bundle exec rake geocombine:clone
+```
+
 #### Clone OpenGeoMetadata repositories locally
 
 ```sh
@@ -63,7 +101,7 @@ You can also specify a single repository:
 $ bundle exec rake geocombine:clone[edu.stanford.purl]
 ```
 
-*Note: If you are using zsh, you will need to use escape characters in front of the brackets:*
+_Note: If you are using zsh, you will need to use escape characters in front of the brackets:_
 
 ```sh
 $ bundle exec rake geocombine:clone\[edu.stanford.purl\]
@@ -83,7 +121,7 @@ You can also specify a single repository:
 $ bundle exec rake geocombine:pull[edu.stanford.purl]
 ```
 
-*Note: If you are using zsh, you will need to use escape characters in front of the brackets:*
+_Note: If you are using zsh, you will need to use escape characters in front of the brackets:_
 
 ```sh
 $ bundle exec rake geocombine:pull\[edu.stanford.purl\]
@@ -98,23 +136,14 @@ To index into Solr, GeoCombine requires a Solr instance that is running the
 $ bundle exec rake geocombine:index
 ```
 
-Indexes the `geoblacklight.json` files in cloned repositories to a Solr index running at http://127.0.0.1:8983/solr
+If Blacklight is installed in the ruby environment and a solr index is configured, the rake task will use the solr index configured in the Blacklight application (this is the case when invoking GeoCombine from your GeoBlacklight installation). If Blacklight is unavailable, the rake task will try to find a Solr instance running at `http://localhost:8983/solr/blacklight-core`.
 
-##### Custom Solr location
-
-Solr location can also be specified by an environment variable `SOLR_URL`.
+You can also set a the Solr instance URL using `SOLR_URL`:
 
 ```sh
 $ SOLR_URL=http://www.example.com:1234/solr/collection bundle exec rake geocombine:index
 ```
 
-Depending on your Solr instance's performance characteristics, you may want to
-change the [`commitWithin` parameter](https://lucene.apache.org/solr/guide/6_6/updatehandlers-in-solrconfig.html) (in milliseconds):
-
-```sh
-$ SOLR_COMMIT_WITHIN=100 bundle exec rake geocombine:index
-```
-
 ### Harvesting and indexing documents from GeoBlacklight sites
 
 GeoCombine provides a Harvester class and rake task to harvest and index content from GeoBlacklight sites (or any site that follows the Blacklight API format). Given that the configurations can change from consumer to consumer and site to site, the class provides a relatively simple configuration API. This can be configured in an initializer, a wrapping rake task, or any other ruby context where the rake task our class would be invoked.
@@ -160,10 +189,6 @@ Crawl delays can be configured (in seconds) either globally for all sites or on
 
 Solr's commitWithin option can be configured (in milliseconds) by passing a value under the commit_within key.
 
-##### Debugging (default: false)
-
-The harvester and indexer will only `puts` content when errors happen. It is possible to see some progress information by setting the debug configuration option.
-
 #### Transforming Documents
 
 You may need to transform documents that are harvested for various purposes (removing fields, adding fields, omitting a document all together, etc). You can configure some ruby code (a proc) that will take the document in, transform it, and return the transformed document. By default the indexer will remove the `score`, `timestamp`, and `_version_` fields from the documents harvested. If you provide your own transformer, you'll likely want to remove these fields in addition to the other transformations you provide.

data/geo_combine.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'thor'
   spec.add_dependency 'faraday-net_http_persistent', '~> 2.0'
   spec.add_dependency 'git'
+  spec.add_dependency 'faraday-retry', '~> 2.2'
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'

data/lib/geo_combine/ckan_metadata.rb

@@ -44,7 +44,8 @@ module GeoCombine
     def envelope
       return envelope_from_bbox unless envelope_from_bbox.nil?
       return envelope_from_spatial(',') unless envelope_from_spatial(',').nil?
-      return envelope_from_spatial(' ') unless envelope_from_spatial(' ').nil?
+
+      envelope_from_spatial(' ') unless envelope_from_spatial(' ').nil?
     end
 
     def envelope_from_bbox
@@ -55,7 +56,7 @@ module GeoCombine
         north: extras('bbox-north-lat')
       )
       begin
-        return bbox.to_envelope if bbox.valid?
+        bbox.to_envelope if bbox.valid?
       rescue GeoCombine::Exceptions::InvalidGeometry
         nil
       end
@@ -64,10 +65,10 @@ module GeoCombine
     def envelope_from_spatial(delimiter)
       bbox = GeoCombine::BoundingBox.from_string_delimiter(
         extras('spatial'),
-        delimiter: delimiter
+        delimiter:
       )
       begin
-        return bbox.to_envelope if bbox.valid?
+        bbox.to_envelope if bbox.valid?
       rescue GeoCombine::Exceptions::InvalidGeometry
         nil
       end

data/lib/geo_combine/geo_blacklight_harvester.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'geo_combine/logger'
+
 module GeoCombine
   ##
   # A class to harvest and index results from GeoBlacklight sites
@@ -45,24 +47,25 @@ module GeoCombine
 
     attr_reader :site, :site_key
 
-    def initialize(site_key)
+    def initialize(site_key, logger: GeoCombine::Logger.logger)
       @site_key = site_key
       @site = self.class.config[site_key]
+      @logger = logger
 
       raise ArgumentError, "Site key #{@site_key.inspect} is not configured for #{self.class.name}" unless @site
     end
 
     def index
-      puts "Fetching page 1 @ #{base_url}&page=1" if self.class.config[:debug]
+      @logger.debug "fetching page 1 @ #{base_url}&page=1"
       response = JSON.parse(Net::HTTP.get(URI("#{base_url}&page=1")))
       response_class = BlacklightResponseVersionFactory.call(response)
 
-      response_class.new(response: response, base_url: base_url).documents.each do |docs|
+      response_class.new(response:, base_url:, logger: @logger).documents.each do |docs|
         docs.map! do |document|
           self.class.document_transformer&.call(document)
         end.compact
 
-        puts "Adding #{docs.count} documents to solr" if self.class.config[:debug]
+        @logger.debug "adding #{docs.count} documents to solr"
         solr_connection.update params: { commitWithin: commit_within, overwrite: true },
                                data: docs.to_json,
                                headers: { 'Content-Type' => 'application/json' }
@@ -91,10 +94,11 @@ module GeoCombine
       attr_reader :base_url
       attr_accessor :response, :page
 
-      def initialize(response:, base_url:)
+      def initialize(response:, base_url:, logger: GeoCombine::Logger.logger)
         @base_url = base_url
         @response = response
         @page = 1
+        @logger = logger
       end
 
       def documents
@@ -106,12 +110,12 @@ module GeoCombine
           break if current_page == total_pages
 
           self.page += 1
-          puts "Fetching page #{page} @ #{url}" if GeoCombine::GeoBlacklightHarvester.config[:debug]
+          @logger.debug "fetching page #{page} @ #{url}"
 
           begin
             self.response = JSON.parse(Net::HTTP.get(URI(url)))
           rescue StandardError => e
-            puts "Request for #{url} failed with #{e}"
+            @logger.error "request for #{url} failed with #{e}"
             self.response = nil
           end
         end
@@ -138,10 +142,11 @@ module GeoCombine
       attr_reader :base_url
       attr_accessor :response, :page
 
-      def initialize(response:, base_url:)
+      def initialize(response:, base_url:, logger: GeoCombine::Logger.logger)
         @base_url = base_url
         @response = response
         @page = 1
+        @logger = logger
       end
 
       def documents
@@ -157,11 +162,11 @@ module GeoCombine
 
           url = "#{url}&format=json"
           self.page += 1
-          puts "Fetching page #{page} @ #{url}" if GeoCombine::GeoBlacklightHarvester.config[:debug]
+          @logger.debug "fetching page #{page} @ #{url}"
           begin
             self.response = JSON.parse(Net::HTTP.get(URI(url)))
           rescue StandardError => e
-            puts "Request for #{url} failed with #{e}"
+            @logger.error "Request for #{url} failed with #{e}"
             self.response = nil
           end
         end
@@ -170,11 +175,11 @@ module GeoCombine
       private
 
       def documents_from_urls(urls)
-        puts "Fetching #{urls.count} documents for page #{page}" if GeoCombine::GeoBlacklightHarvester.config[:debug]
+        @logger.debug "fetching #{urls.count} documents for page #{page}"
         urls.map do |url|
           JSON.parse(Net::HTTP.get(URI("#{url}/raw")))
         rescue StandardError => e
-          puts "Fetching \"#{url}/raw\" failed with #{e}"
+          @logger.error "fetching \"#{url}/raw\" failed with #{e}"
 
           nil
         end.compact

data/lib/geo_combine/geoblacklight.rb

@@ -13,7 +13,7 @@ module GeoCombine
     attr_reader :metadata
 
     GEOBLACKLIGHT_VERSION = '1.0'
-    SCHEMA_JSON_URL = "https://raw.githubusercontent.com/OpenGeoMetadata/opengeometadata.github.io/main/docs/schema/geoblacklight-schema-#{GEOBLACKLIGHT_VERSION}.json"
+    SCHEMA_JSON_URL = "https://raw.githubusercontent.com/OpenGeoMetadata/opengeometadata.github.io/main/docs/schema/geoblacklight-schema-#{GEOBLACKLIGHT_VERSION}.json".freeze
     DEPRECATED_KEYS_V1 = %w[
       uuid
       georss_polygon_s

data/lib/geo_combine/harvester.rb

@@ -3,6 +3,8 @@
 require 'json'
 require 'find'
 require 'git'
+require 'net/http'
+require 'geo_combine/logger'
 
 module GeoCombine
   # Harvests Geoblacklight documents from OpenGeoMetadata for indexing
@@ -29,26 +31,37 @@ module GeoCombine
 
     def initialize(
       ogm_path: ENV.fetch('OGM_PATH', 'tmp/opengeometadata'),
-      schema_version: ENV.fetch('SCHEMA_VERSION', '1.0')
+      schema_version: ENV.fetch('SCHEMA_VERSION', '1.0'),
+      logger: GeoCombine::Logger.logger
     )
       @ogm_path = ogm_path
       @schema_version = schema_version
+      @logger = logger
     end
 
     # Enumerable of docs to index, for passing to an indexer
     def docs_to_index
       return to_enum(:docs_to_index) unless block_given?
 
+      @logger.info "loading documents from #{ogm_path}"
       Find.find(@ogm_path) do |path|
         # skip non-json and layers.json files
-        next unless File.basename(path).include?('.json') && File.basename(path) != 'layers.json'
+        if File.basename(path) == 'layers.json' || !File.basename(path).end_with?('.json')
+          @logger.debug "skipping #{path}; not a geoblacklight JSON document"
+          next
+        end
 
         doc = JSON.parse(File.read(path))
         [doc].flatten.each do |record|
           # skip indexing if this record has a different schema version than what we want
           record_schema = record['gbl_mdVersion_s'] || record['geoblacklight_version']
-          next unless record_schema == @schema_version
+          record_id = record['layer_slug_s'] || record['dc_identifier_s']
+          if record_schema != @schema_version
+            @logger.debug "skipping #{record_id}; schema version #{record_schema} doesn't match #{@schema_version}"
+            next
+          end
 
+          @logger.debug "found record #{record_id} at #{path}"
           yield record, path
         end
       end
@@ -61,14 +74,16 @@ module GeoCombine
       clone(repo) unless File.directory? repo_path
 
       Git.open(repo_path).pull
-      puts "Updated #{repo}"
-      1
+      @logger.info "updated #{repo}"
+      repo
     end
 
     # Update all repositories
-    # Return the count of repositories updated
+    # Return the names of repositories updated
     def pull_all
-      repositories.map(&method(:pull)).reduce(:+)
+      updated = repositories.map(&method(:pull)).compact
+      @logger.info "updated #{updated.size} repositories"
+      updated
     end
 
     # Clone a repository via git
@@ -76,25 +91,27 @@ module GeoCombine
     def clone(repo)
       repo_path = File.join(@ogm_path, repo)
       repo_info = repository_info(repo)
+      repo_url = "https://github.com/OpenGeoMetadata/#{repo}.git"
 
       # Skip if exists; warn if archived or empty
       if File.directory? repo_path
-        puts "Skipping clone to #{repo_path}; directory exists"
-        return 0
+        @logger.warn "skipping clone to #{repo_path}; directory exists"
+        return nil
       end
-      puts "WARNING: repository '#{repo}' is archived" if repo_info['archived']
-      puts "WARNING: repository '#{repo}' is empty" if repo_info['size'].zero?
+      @logger.warn "repository is archived: #{repo_url}" if repo_info['archived']
+      @logger.warn "repository is empty: #{repo_url}" if repo_info['size'].zero?
 
-      repo_url = "https://github.com/OpenGeoMetadata/#{repo}.git"
       Git.clone(repo_url, nil, path: ogm_path, depth: 1)
-      puts "Cloned #{repo_url}"
-      1
+      @logger.info "cloned #{repo_url} to #{repo_path}"
+      repo
     end
 
     # Clone all repositories via git
-    # Return the count of repositories cloned.
+    # Return the names of repositories cloned.
     def clone_all
-      repositories.map(&method(:clone)).reduce(:+)
+      cloned = repositories.map(&method(:clone)).compact
+      @logger.info "cloned #{cloned.size} repositories"
+      cloned
     end
 
     private