geo_combine 0.9.2 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f3486b76e27b3a26ad35ed97c2ed196890ea7ea0a4f707f8cfebca3967cd1472
-  data.tar.gz: 0c2d1e07f52144bd96faec67b83facb0a379fbc17f69fcaa36fd584da5b32773
+  metadata.gz: cf25387203a718f8ae41365d87fae7d1868ea1416bf8ae9a33d11b6fef21171d
+  data.tar.gz: 7cbdec148a45206e21090b123390ac129d705c502e2ed1bd2c0d723b81afd51c
 SHA512:
-  metadata.gz: 71a0cc0aa25959868ef0eb370d8b53561d3505ee3bdf3c8dbd3d8a1553ff66c1049eed1521ea9ba4e665a615c36712a6dd3d025488ab8e46e1c91b2b0536e4f3
-  data.tar.gz: 95422cfd7054c883a2fad7b31083cd514eb4cecb4ebd978f6df692d35e801484d71b78f4e316a358ad18f67ceb0e13bb1fcb52f6cfee5b10b5c48747294d693b
+  metadata.gz: 1319f17633d5ee99fe3706dcdae9b7a327339b658ee8677b678aa23132b8b48b031589b217399c53078625b5a3f19cbdd11f13a00313edbe563a0ea8b04c651b
+  data.tar.gz: 521143da86eabc8ec0d89a45c82d8ccf58f2ab6bab03bf06357fd124e3601898a2ee8cd9a522b5d7a126d0e106e93f4790e84fd915186439eec7a7956814cf7c

data/.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

data/.github/workflows/ruby.yml

@@ -6,11 +6,11 @@ jobs:
   rubocop:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v6
     - name: Set up Ruby and install dependencies
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 3.1
+        ruby-version: 3.2
         bundler-cache: true
     - name: Run linter
       run: bundle exec rubocop
@@ -19,10 +19,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby: [3.1, 3.2, 3.3]
+        ruby: [3.2, 3.3, 3.4]
         faraday_version: ['', '~> 1.0'] # Defaults to whatever's the most recent version.
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v6
 
     - name: Set up Ruby and install dependencies
       uses: ruby/setup-ruby@v1
@@ -32,13 +32,13 @@ jobs:
       env:
         FARADAY_VERSION: ${{ matrix.faraday_version }}
 
-    - name: Run tests
+    - name: "Run tests for Ruby ${{ matrix.ruby }} and Faraday ${{ matrix.faraday_version == '~> 1.0' && '1.x' || 'latest' }}"
       run: bundle exec rake spec
       env:
         FARADAY_VERSION: ${{ matrix.faraday_version }}
 
     - name: Upload coverage artifacts
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v6
       with:
-        name: coverage
+        name: "coverage-${{ matrix.ruby }}-${{ matrix.faraday_version == '~> 1.0' && 'faraday-1.x' || 'faraday-latest' }}"
         path: coverage/

data/.gitignore

@@ -15,3 +15,4 @@ mkmf.log
 .tool-versions
 .byebug_history
 .ruby-version
+.DS_Store

data/.rubocop.yml

@@ -1,5 +1,7 @@
 require:
   - rubocop-rspec
+
+plugins:
   - rubocop-rake
 
 inherit_from: .rubocop_todo.yml

data/.rubocop_todo.yml

@@ -41,7 +41,7 @@ Metrics/CyclomaticComplexity:
 # Offense count: 13
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 Metrics/MethodLength:
-  Max: 21
+  Max: 25
 
 # Offense count: 2
 # Configuration parameters: AllowedMethods, AllowedPatterns.

data/Gemfile

@@ -5,4 +5,5 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in geo_combine.gemspec
 gemspec
 
-gem 'byebug'
+gem 'byebug', require: false
+gem 'rdbg', require: false

data/README.md

@@ -32,8 +32,11 @@ $ gem install geo_combine
 ## Usage
 
 ### Converting metadata
+
 #### Converting metadata into GeoBlacklight JSON
+
 GeoCombine provides several classes representing different metadata standards that implement the `#to_geoblacklight` method for generating records in the [GeoBlacklight JSON format](https://opengeometadata.org/reference/):
+
 ```ruby
 GeoCombine::Iso19139 # ISO 19139 XML
 GeoCombine::OGP # OpenGeoPortal JSON
@@ -41,7 +44,9 @@ GeoCombine::Fgdc # FGDC XML
 GeoCombine::EsriOpenData # Esri Open Data Portal JSON
 GeoCombine::CkanMetadata # CKAN JSON
 ```
+
 An example for converting an ISO 19139 XML record:
+
 ```ruby
 # Create a new ISO19139 object
 > iso_metadata =  GeoCombine::Iso19139.new('./tmp/opengeometadata/edu.stanford.purl/bb/338/jh/0716/iso19139.xml')
@@ -52,7 +57,9 @@ An example for converting an ISO 19139 XML record:
 # Output it as JSON instead of a Ruby hash
 > iso_metadata.to_geoblacklight.to_json
 ```
+
 Some formats also support conversion into HTML for display in a web browser:
+
 ```ruby
 # Create a new ISO19139 object
 > iso_metadata =  GeoCombine::Iso19139.new('./tmp/opengeometadata/edu.stanford.purl/bb/338/jh/0716/iso19139.xml')
@@ -94,7 +101,16 @@ GeoCombine::Migrators::V1AardvarkMigrator.new(v1_hash: record, collection_id_map
 Some of the tools and scripts in this gem use Ruby's `Logger` class to print information to `$stderr`. By default, the log level is set to `Logger::INFO`. For more verbose information, you can set the `LOG_LEVEL` environment variable to `DEBUG`:
 
 ```sh
-$ LOG_LEVEL=DEBUG bundle exec rake geocombine:clone
+$ export LOG_LEVEL=DEBUG
+```
+
+#### Schema version
+
+By default, GeoCombine will only fetch and index records using the current schema version. If you instead want to index records using an older schema (e.g. because your GeoBlacklight instance is an older version), you can set the `SCHEMA_VERSION` environment variable:
+
+```sh
+# Only fetch and index schema version 1.0 records
+$ export SCHEMA_VERSION=1.0
 ```
 
 #### Clone OpenGeoMetadata repositories locally
@@ -103,7 +119,7 @@ $ LOG_LEVEL=DEBUG bundle exec rake geocombine:clone
 $ bundle exec rake geocombine:clone
 ```
 
-Will clone all `edu.*`,` org.*`, and `uk.*` OpenGeoMetadata repositories into `./tmp/opengeometadata`. Location of the OpenGeoMetadata repositories can be configured using the `OGM_PATH` environment variable.
+Will clone all OpenGeoMetadata repositories containing metadata matching the `SCHEMA_VERSION` into `./tmp/opengeometadata`. Location of the OpenGeoMetadata repositories can be configured using the `OGM_PATH` environment variable.
 
 ```sh
 $ OGM_PATH='my/custom/location' bundle exec rake geocombine:clone
@@ -152,17 +168,28 @@ $ bundle exec rake geocombine:index
 
 If Blacklight is installed in the ruby environment and a solr index is configured, the rake task will use the solr index configured in the Blacklight application (this is the case when invoking GeoCombine from your GeoBlacklight installation). If Blacklight is unavailable, the rake task will try to find a Solr instance running at `http://localhost:8983/solr/blacklight-core`.
 
-You can also set a the Solr instance URL using `SOLR_URL`:
+You can also set the Solr instance URL using `SOLR_URL`:
 
 ```sh
 $ SOLR_URL=http://www.example.com:1234/solr/collection bundle exec rake geocombine:index
 ```
 
-By default, GeoCombine will index only records using the Aardvark metadata format. If you instead want to index records using an older format (e.g. because your GeoBlacklight instance is version 3 or older), you can set the `SCHEMA_VERSION` environment variable:
+### Indexing local documents
+
+To index an arbitrary collection of records in a custom directory, run one of the following:
+
+```sh
+rake geocombine:index\[/path/to/your/files\]
+OGM_PATH=/path/to/your/files rake geocombine:index
+```
+
+### Indexing a single document
+
+To index a single arbitrary record, run one of the following:
 
 ```sh
-# Only index schema version 1.0 records
-$ SCHEMA_VERSION=1.0 bundle exec rake geocombine:index
+rake geocombine:index\[/path/to/your/file.json\]
+OGM_PATH=/path/to/your/file.json rake geocombine:index
 ```
 
 ### Harvesting and indexing documents from GeoBlacklight sites

data/geo_combine.gemspec

@@ -32,8 +32,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rspec-html-matchers'
-  spec.add_development_dependency 'rubocop', '~> 1.25'
-  spec.add_development_dependency 'rubocop-rspec', '~> 2.8'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'rubocop-rspec'
   spec.add_development_dependency 'rubocop-rake'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'webmock', '~> 3.14'

data/lib/geo_combine/bounding_box.rb

@@ -48,8 +48,6 @@ module GeoCombine
     end
 
     def self.from_envelope(envelope)
-      return if envelope.nil?
-
       envelope = envelope[/.*ENVELOPE\(([^)]*)/, 1].split(',')
       new(
         west: envelope[0],
@@ -63,8 +61,6 @@ module GeoCombine
     # @param [String] spatial w,s,e,n or w s e n
     # @param [String] delimiter "," or " "
     def self.from_string_delimiter(spatial, delimiter: ',')
-      return if spatial.nil?
-
       spatial = spatial.split(delimiter)
       new(
         west: spatial[0],

data/lib/geo_combine/ckan_metadata.rb

@@ -32,7 +32,7 @@ module GeoCombine
         layer_slug_s: @metadata['name'],
         solr_geom: envelope,
         dc_subject_sm: subjects,
-        dct_references_s: external_references.to_json.to_s,
+        dct_references_s: external_references.to_json,
         dc_format_s: downloadable? ? 'ZIP' : nil # TODO: we only allow direct ZIP file downloads
       }.compact
     end

data/lib/geo_combine/esri_open_data.rb

@@ -4,6 +4,7 @@ module GeoCombine
   # Data model for ESRI's open data portal metadata
   class EsriOpenData
     include GeoCombine::Formatting
+
     attr_reader :metadata
 
     ##

data/lib/geo_combine/geoblacklight.rb

@@ -42,6 +42,7 @@ module GeoCombine
       metadata.each do |key, value|
         translate_formats(key, value)
         enhance_subjects(key, value)
+        infer_geometry_type_from_subject(key, value)
         format_proper_date(key, value)
         fields_should_be_array(key, value)
         translate_geometry_type(key, value)
@@ -55,20 +56,29 @@ module GeoCombine
       metadata.to_json(options)
     end
 
+    # True if the GeoBlacklight-Schema document is valid
+    # @return [Boolean]
+    def valid?
+      validate!
+      true
+    rescue StandardError
+      false
+    end
+
     ##
     # Validates a GeoBlacklight-Schema json document
     # @return [Boolean]
-    def valid?
-      JSON::Validator.validate!(schema, to_json, fragment: '#/definitions/layer') &&
-        dct_references_validate! &&
-        spatial_validate!
+    def validate!
+      JSON::Validator.validate!(schema, to_json, fragment: '#/definitions/layer')
+      validate_references!
+      validate_spatial!
     end
 
     ##
     # Validate dct_references_s
     # @return [Boolean]
-    def dct_references_validate!
-      return true unless metadata.key?('dct_references_s') # TODO: shouldn't we require this field?
+    def validate_references!
+      return unless metadata.key?('dct_references_s') # Not required
 
       begin
         ref = JSON.parse(metadata['dct_references_s'])
@@ -83,8 +93,8 @@ module GeoCombine
       end
     end
 
-    def spatial_validate!
-      GeoCombine::BoundingBox.from_envelope(metadata['solr_geom']).valid?
+    def validate_spatial!
+      raise GeoCombine::Exceptions::InvalidGeometry unless GeoCombine::BoundingBox.from_envelope(metadata['solr_geom']).valid?
     end
 
     private
@@ -106,6 +116,17 @@ module GeoCombine
       metadata[key] = geometry_types[value]
     end
 
+    ##
+    # Enhances empty 'layer_geom_type_s' field by populating from related subject
+    def infer_geometry_type_from_subject(key, value)
+      return unless key == 'layer_geom_type_s' && value.blank?
+
+      # If any subjects match a known geometry type, use the first that does as the geometry type
+      metadata['dc_subject_sm'].find { |subject| geometry_types.value?(subject.capitalize) }&.tap do |subject|
+        metadata[key] = subject.capitalize
+      end
+    end
+
     ##
     # Enhances the 'dc_subject_sm' field by translating subjects to ISO topic
     # categories
@@ -163,7 +184,7 @@ module GeoCombine
       metadata.except!(*DEPRECATED_KEYS_V1)
 
       # ensure we have a proper v1 record
-      valid?
+      validate!
     end
 
     def schema

data/lib/geo_combine/harvester.rb

@@ -11,31 +11,25 @@ module GeoCombine
   class Harvester
     attr_reader :ogm_path, :schema_version
 
-    # Non-metadata repositories that shouldn't be harvested
-    def self.denylist
-      [
-        'GeoCombine',
-        'aardvark',
-        'metadata-issues',
-        'ogm_utils-python',
-        'opengeometadata.github.io',
-        'opengeometadata-rails',
-        'gbl-1_to_aardvark'
-      ]
-    end
-
     # GitHub API endpoint for OpenGeoMetadata repositories
     def self.ogm_api_uri
       URI('https://api.github.com/orgs/opengeometadata/repos?per_page=1000')
     end
 
+    # Initialize a new harvester
+    # @param ogm_path [String] path to the directory where repositories will be cloned
+    # @param skip_repos [Array<String>] list of repository names to skip
+    # @param schema_version [String] schema version to filter repositories by
+    # @param logger [Logger] logger to use for logging messages
     def initialize(
-      ogm_path: ENV.fetch('OGM_PATH', 'tmp/opengeometadata'),
+      ogm_path:,
+      skip_repos: ENV.fetch('OGM_SKIP_REPOS', '').split(',').map(&:strip),
       schema_version: ENV.fetch('SCHEMA_VERSION', 'Aardvark'),
       logger: GeoCombine::Logger.logger
     )
       @ogm_path = ogm_path
       @schema_version = schema_version
+      @skip_repos = skip_repos
       @logger = logger
     end
 
@@ -53,9 +47,16 @@ module GeoCombine
 
         doc = JSON.parse(File.read(path))
         [doc].flatten.each do |record|
-          # skip indexing if this record has a different schema version than what we want
           record_schema = record['gbl_mdVersion_s'] || record['geoblacklight_version']
           record_id = record['layer_slug_s'] || record['dc_identifier_s']
+
+          # skip indexing if no identifiable schema version
+          unless record_schema
+            @logger.debug "skipping #{record_id || path}; no schema version declared in record"
+            next
+          end
+
+          # skip indexing if this record has a different schema version than what we want
           if record_schema != @schema_version
             @logger.debug "skipping #{record_id}; schema version #{record_schema} doesn't match #{@schema_version}"
             next
@@ -87,19 +88,20 @@ module GeoCombine
     end
 
     # Clone a repository via git
-    # If the repository already exists, skip it.
+    # Return the name of the repository cloned, or nil if skipped
     def clone(repo)
       repo_path = File.join(@ogm_path, repo)
       repo_info = repository_info(repo)
       repo_url = "https://github.com/OpenGeoMetadata/#{repo}.git"
-
-      # Skip if exists; warn if archived or empty
-      if File.directory? repo_path
-        @logger.warn "skipping clone to #{repo_path}; directory exists"
-        return nil
+      repo_schemas = Array(repo_info.dig('custom_properties', 'supported_schemas'))
+
+      # Skip if exists, archived, empty, or different schema
+      return @logger.warn "skipping clone to #{repo_path}; directory exists" if File.directory? repo_path
+      return @logger.warn "repository is archived: #{repo_url}" if repo_info['archived']
+      return @logger.warn "repository is empty: #{repo_url}" if repo_info['size'].zero?
+      unless repo_schemas.include? @schema_version
+        return @logger.warn "repository #{repo_url} clone to #{repo_path}; repository properties don't include schema version #{@schema_version} (found #{repo_schemas.join(', ')})"
       end
-      @logger.warn "repository is archived: #{repo_url}" if repo_info['archived']
-      @logger.warn "repository is empty: #{repo_url}" if repo_info['size'].zero?
 
       Git.clone(repo_url, nil, path: ogm_path, depth: 1)
       @logger.info "cloned #{repo_url} to #{repo_path}"
@@ -119,12 +121,14 @@ module GeoCombine
     # List of repository names to harvest
     def repositories
       @repositories ||= JSON.parse(Net::HTTP.get(self.class.ogm_api_uri))
+                            .filter { |repo| Array(repo.dig('custom_properties', 'supported_schemas')).include? @schema_version }
                             .filter { |repo| repo['size'].positive? }
                             .reject { |repo| repo['archived'] }
+                            .reject { |repo| @skip_repos.include?(repo['name']) }
                             .map { |repo| repo['name'] }
-                            .reject { |name| self.class.denylist.include? name }
     end
 
+    # Fetch repository metadata from GitHub API
     def repository_info(repo_name)
       JSON.parse(Net::HTTP.get(URI("https://api.github.com/repos/opengeometadata/#{repo_name}")))
     end

data/lib/geo_combine/ogp.rb

@@ -8,6 +8,7 @@ module GeoCombine
   class OGP
     class InvalidMetadata < RuntimeError; end
     include GeoCombine::Formatting
+
     attr_reader :metadata
 
     ##
@@ -123,7 +124,7 @@ module GeoCombine
     # Builds a Solr Envelope using CQL syntax
     # @return [String]
     def envelope
-      raise ArgumentError unless west >= -180 && west <= 180 &&
+      raise ArgumentError unless west.between?(-180, 180) &&
                                  east >= -180 && east <= 180 &&
                                  north >= -90 && north <= 90 &&
                                  south >= -90 && south <= 90 &&
@@ -144,6 +145,15 @@ module GeoCombine
       GeoCombine::Fgdc.new(metadata['FgdcText']) if metadata['FgdcText']
     end
 
+    SLUG_STRIP_VALUES = %w[
+      SDE_DATA.
+      SDE.
+      SDE2.
+      GISPORTAL.GISOWNER01.
+      GISDATA.
+      MORIS.
+    ].freeze
+
     private
 
     ##
@@ -207,15 +217,6 @@ module GeoCombine
       sluggify(filter_name(name))
     end
 
-    SLUG_STRIP_VALUES = %w[
-      SDE_DATA.
-      SDE.
-      SDE2.
-      GISPORTAL.GISOWNER01.
-      GISDATA.
-      MORIS.
-    ].freeze
-
     def filter_name(name)
       # strip out schema and usernames
       SLUG_STRIP_VALUES.each do |strip_val|

data/lib/geo_combine/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GeoCombine
-  VERSION = '0.9.2'
+  VERSION = '0.10.0'
 end

data/lib/tasks/geo_combine.rake

@@ -11,20 +11,26 @@ require 'geo_combine/geo_blacklight_harvester'
 namespace :geocombine do
   desc 'Clone OpenGeoMetadata repositories'
   task :clone, [:repo] do |_t, args|
-    harvester = GeoCombine::Harvester.new
+    ogm_path = ENV.fetch('OGM_PATH', 'tmp/opengeometadata')
+
+    harvester = GeoCombine::Harvester.new(ogm_path: ogm_path)
     args[:repo] ? harvester.clone(args.repo) : harvester.clone_all
   end
 
   desc '"git pull" OpenGeoMetadata repositories'
   task :pull, [:repo] do |_t, args|
-    harvester = GeoCombine::Harvester.new
+    ogm_path = ENV.fetch('OGM_PATH', 'tmp/opengeometadata')
+
+    harvester = GeoCombine::Harvester.new(ogm_path: ogm_path)
     args[:repo] ? harvester.pull(args.repo) : harvester.pull_all
   end
 
   desc 'Index all JSON documents except Layers.json'
-  task :index do
-    harvester = GeoCombine::Harvester.new
+  task :index, [:ogm_path] => [:environment] do |_t, args|
+    ogm_path = args.ogm_path || ENV.fetch('OGM_PATH', 'tmp/opengeometadata')
+
     indexer = GeoCombine::Indexer.new
+    harvester = GeoCombine::Harvester.new(ogm_path: ogm_path)
     indexer.index(harvester.docs_to_index)
   end