geo_combine 0.9.2 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f3486b76e27b3a26ad35ed97c2ed196890ea7ea0a4f707f8cfebca3967cd1472
-  data.tar.gz: 0c2d1e07f52144bd96faec67b83facb0a379fbc17f69fcaa36fd584da5b32773
+  metadata.gz: 21614ec82bcbe5c5a7bb1291e398795f1baff048552746335f18f315da677c9e
+  data.tar.gz: 6c94a18a0f43fd141f5967930a909efac60ab560ac91a0759e48cc45fdab39b5
 SHA512:
-  metadata.gz: 71a0cc0aa25959868ef0eb370d8b53561d3505ee3bdf3c8dbd3d8a1553ff66c1049eed1521ea9ba4e665a615c36712a6dd3d025488ab8e46e1c91b2b0536e4f3
-  data.tar.gz: 95422cfd7054c883a2fad7b31083cd514eb4cecb4ebd978f6df692d35e801484d71b78f4e316a358ad18f67ceb0e13bb1fcb52f6cfee5b10b5c48747294d693b
+  metadata.gz: 75ce7b76dfa6ad5905410e6b38a18ee98185c50d5d116c8fc5794eba3376e85a05d2e2ad7d70d6bde8de7b0073df22f15cb28759633e0d3c6e546e24a13dee82
+  data.tar.gz: cbd5ccc9eac444e599e0dce59fda1f23ef752cec71508a83fbe32ea1b2785b5a8c52d66c4ebcebc81a913aaccf6f677f5555622a43a8e16e902ab5f3379cbfa0

data/.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

data/.github/workflows/ruby.yml

@@ -6,11 +6,11 @@ jobs:
   rubocop:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v6
     - name: Set up Ruby and install dependencies
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 3.1
+        ruby-version: 3.2
         bundler-cache: true
     - name: Run linter
       run: bundle exec rubocop
@@ -19,10 +19,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby: [3.1, 3.2, 3.3]
+        ruby: [3.2, 3.3, 3.4]
         faraday_version: ['', '~> 1.0'] # Defaults to whatever's the most recent version.
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v6
 
     - name: Set up Ruby and install dependencies
       uses: ruby/setup-ruby@v1
@@ -32,13 +32,13 @@ jobs:
       env:
         FARADAY_VERSION: ${{ matrix.faraday_version }}
 
-    - name: Run tests
+    - name: "Run tests for Ruby ${{ matrix.ruby }} and Faraday ${{ matrix.faraday_version == '~> 1.0' && '1.x' || 'latest' }}"
       run: bundle exec rake spec
       env:
         FARADAY_VERSION: ${{ matrix.faraday_version }}
 
     - name: Upload coverage artifacts
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v6
       with:
-        name: coverage
+        name: "coverage-${{ matrix.ruby }}-${{ matrix.faraday_version == '~> 1.0' && 'faraday-1.x' || 'faraday-latest' }}"
         path: coverage/

data/.gitignore

@@ -15,3 +15,4 @@ mkmf.log
 .tool-versions
 .byebug_history
 .ruby-version
+.DS_Store

data/.rubocop.yml

@@ -1,5 +1,7 @@
 require:
   - rubocop-rspec
+
+plugins:
   - rubocop-rake
 
 inherit_from: .rubocop_todo.yml

data/.rubocop_todo.yml

@@ -41,7 +41,7 @@ Metrics/CyclomaticComplexity:
 # Offense count: 13
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 Metrics/MethodLength:
-  Max: 21
+  Max: 25
 
 # Offense count: 2
 # Configuration parameters: AllowedMethods, AllowedPatterns.

data/Gemfile

@@ -5,4 +5,5 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in geo_combine.gemspec
 gemspec
 
-gem 'byebug'
+gem 'byebug', require: false
+gem 'rdbg', require: false

data/README.md

@@ -1,16 +1,20 @@
 # GeoCombine
 
 ![CI](https://github.com/OpenGeoMetadata/GeoCombine/actions/workflows/ruby.yml/badge.svg)
-| [![Coverage Status](https://img.shields.io/badge/coverage-95%25-brightgreen)]()
-| [![Gem Version](https://img.shields.io/gem/v/geo_combine.svg)](https://github.com/OpenGeoMetadata/GeoCombine/releases)
+[![Coverage Status](https://img.shields.io/badge/coverage-95%25-brightgreen)]()
+[![Gem Version](https://img.shields.io/gem/v/geo_combine.svg)](https://github.com/OpenGeoMetadata/GeoCombine/releases)
 
 A Ruby toolkit for managing geospatial metadata, including:
 
-- tasks for cloning, updating, and indexing OpenGeoMetadata metadata
-- library for converting metadata between standards
+- harvesting and indexing records from OpenGeoMetadata
+- harvesting records directly from a GeoBlacklight instance
+- converting geospatial metadata records between standards
 
 ## Installation
 
+> [!TIP]
+> If you're using GeoBlacklight, GeoCombine is already bundled with your installation – you can skip this section.
+
 Add this line to your application's `Gemfile`:
 
 ```ruby
@@ -20,111 +24,64 @@ gem 'geo_combine'
 And then execute:
 
 ```sh
-$ bundle install
+bundle install
 ```
 
 Or install it yourself as:
 
 ```sh
-$ gem install geo_combine
+gem install geo_combine
 ```
 
 ## Usage
 
-### Converting metadata
-#### Converting metadata into GeoBlacklight JSON
-GeoCombine provides several classes representing different metadata standards that implement the `#to_geoblacklight` method for generating records in the [GeoBlacklight JSON format](https://opengeometadata.org/reference/):
-```ruby
-GeoCombine::Iso19139 # ISO 19139 XML
-GeoCombine::OGP # OpenGeoPortal JSON
-GeoCombine::Fgdc # FGDC XML
-GeoCombine::EsriOpenData # Esri Open Data Portal JSON
-GeoCombine::CkanMetadata # CKAN JSON
-```
-An example for converting an ISO 19139 XML record:
-```ruby
-# Create a new ISO19139 object
-> iso_metadata =  GeoCombine::Iso19139.new('./tmp/opengeometadata/edu.stanford.purl/bb/338/jh/0716/iso19139.xml')
-
-# Convert to GeoBlacklight's metadata format
-> iso_metadata.to_geoblacklight
-
-# Output it as JSON instead of a Ruby hash
-> iso_metadata.to_geoblacklight.to_json
-```
-Some formats also support conversion into HTML for display in a web browser:
-```ruby
-# Create a new ISO19139 object
-> iso_metadata =  GeoCombine::Iso19139.new('./tmp/opengeometadata/edu.stanford.purl/bb/338/jh/0716/iso19139.xml')
-
-# Convert ISO to HTML
-> iso_metadata.to_html
-```
-
-### Migrating metadata
-
-You can use the `GeoCombine::Migrators` to migrate metadata from one schema to another.
-
-Currently, the only migrator is `GeoCombine::Migrators::V1AardvarkMigrator` which migrates from the [GeoBlacklight v1 schema](https://github.com/OpenGeoMetadata/opengeometadata.github.io/blob/main/docs/gbl-1.0.md) to the [Aardvark schema](https://github.com/OpenGeoMetadata/opengeometadata.github.io/blob/main/docs/ogm-aardvark.md)
-
-```ruby
-# Load a record in geoblacklight v1 schema
-record = JSON.parse(File.read('.spec/fixtures/docs/full_geoblacklight.json'))
-
-# Migrate it to Aardvark schema
-GeoCombine::Migrators::V1AardvarkMigrator.new(v1_hash: record).run
-```
+### Downloading metadata from OpenGeoMetadata
 
-Some fields cannot be migrated automatically. To handle the migration of collection names to IDs when migrating from v1 to Aardvark, you can provide a mapping of collection names to IDs to the migrator:
+#### Schema version
 
-```ruby
-# You can store this mapping as a JSON or CSV file and load it into a hash
-id_map = {
-  'My Collection 1' => 'institution:my-collection-1',
-  'My Collection 2' => 'institution:my-collection-2'
-}
+By default, GeoCombine will only fetch and index records using the current schema version. If you instead want to index records using an older schema (e.g. because your GeoBlacklight instance is an older version), you can set the `SCHEMA_VERSION` environment variable:
 
-GeoCombine::Migrators::V1AardvarkMigrator.new(v1_hash: record, collection_id_map: id_map).run
+```sh
+# Only fetch and index schema version 1.0 records
+export SCHEMA_VERSION=1.0
 ```
 
-### Downloading metadata from OpenGeoMetadata
+#### Restricted records
 
-#### Logging
-
-Some of the tools and scripts in this gem use Ruby's `Logger` class to print information to `$stderr`. By default, the log level is set to `Logger::INFO`. For more verbose information, you can set the `LOG_LEVEL` environment variable to `DEBUG`:
+By default, GeoCombine will only fetch and index records that are not marked as restricted. If you want to include restricted records, you can set the `OGM_SKIP_RESTRICTED` environment variable to `false`:
 
 ```sh
-$ LOG_LEVEL=DEBUG bundle exec rake geocombine:clone
+export OGM_SKIP_RESTRICTED=false
 ```
 
 #### Clone OpenGeoMetadata repositories locally
 
 ```sh
-$ bundle exec rake geocombine:clone
+bundle exec rake geocombine:clone
 ```
 
-Will clone all `edu.*`,` org.*`, and `uk.*` OpenGeoMetadata repositories into `./tmp/opengeometadata`. Location of the OpenGeoMetadata repositories can be configured using the `OGM_PATH` environment variable.
+Will clone all OpenGeoMetadata repositories containing metadata matching the `SCHEMA_VERSION` into `./tmp/opengeometadata`. Location of the OpenGeoMetadata repositories can be configured using the `OGM_PATH` environment variable.
 
 ```sh
-$ OGM_PATH='my/custom/location' bundle exec rake geocombine:clone
+OGM_PATH='my/custom/location' bundle exec rake geocombine:clone
 ```
 
 You can also specify a single repository:
 
 ```sh
-$ bundle exec rake geocombine:clone[edu.stanford.purl]
+bundle exec rake geocombine:clone[edu.stanford.purl]
 ```
 
 _Note: If you are using zsh, you will need to use escape characters in front of the brackets:_
 
 ```sh
-$ bundle exec rake geocombine:clone\[edu.stanford.purl\]
+bundle exec rake geocombine:clone\[edu.stanford.purl\]
 ```
 
 #### Update local OpenGeoMetadata repositories
 
 ```sh
-$ bundle exec rake geocombine:pull
+bundle exec rake geocombine:pull
 ```
 
 Runs `git pull origin master` on all cloned repositories in `./tmp/opengeometadata` (or custom path with configured environment variable `OGM_PATH`).
@@ -132,37 +89,67 @@ Runs `git pull origin master` on all cloned repositories in `./tmp/opengeometada
 You can also specify a single repository:
 
 ```sh
-$ bundle exec rake geocombine:pull[edu.stanford.purl]
+bundle exec rake geocombine:pull[edu.stanford.purl]
 ```
 
 _Note: If you are using zsh, you will need to use escape characters in front of the brackets:_
 
 ```sh
-$ bundle exec rake geocombine:pull\[edu.stanford.purl\]
+bundle exec rake geocombine:pull\[edu.stanford.purl\]
 ```
 
 #### Index GeoBlacklight documents
 
+> [!TIP]
+> If you're using GeoBlacklight, GeoCombine will automatically detect the path to your Solr index if you run it in the directory where GeoBlacklight is installed.
+
 To index into Solr, GeoCombine requires a Solr instance that is running the
 [GeoBlacklight schema](https://github.com/geoblacklight/geoblacklight):
 
 ```sh
-$ bundle exec rake geocombine:index
+bundle exec rake geocombine:index
 ```
 
 If Blacklight is installed in the ruby environment and a solr index is configured, the rake task will use the solr index configured in the Blacklight application (this is the case when invoking GeoCombine from your GeoBlacklight installation). If Blacklight is unavailable, the rake task will try to find a Solr instance running at `http://localhost:8983/solr/blacklight-core`.
 
-You can also set a the Solr instance URL using `SOLR_URL`:
+You can also set the Solr instance URL using `SOLR_URL`:
+
+```sh
+SOLR_URL=http://www.example.com:1234/solr/collection bundle exec rake geocombine:index
+```
+
+#### Logging
+
+Some of the tools and scripts in this gem use Ruby's `Logger` class to print information to `$stderr`. By default, the log level is set to `Logger::INFO`. For more verbose information, you can set the `LOG_LEVEL` environment variable to `DEBUG`:
+
+```sh
+export LOG_LEVEL=DEBUG
+```
+
+### Indexing local documents
+
+To index a collection of records in a local directory, you can pass the directory as a `rake` argument:
 
 ```sh
-$ SOLR_URL=http://www.example.com:1234/solr/collection bundle exec rake geocombine:index
+bundle exec rake geocombine:index\[/path/to/your/files\]
 ```
 
-By default, GeoCombine will index only records using the Aardvark metadata format. If you instead want to index records using an older format (e.g. because your GeoBlacklight instance is version 3 or older), you can set the `SCHEMA_VERSION` environment variable:
+Or you can provide it as `OGM_PATH`:
 
 ```sh
-# Only index schema version 1.0 records
-$ SCHEMA_VERSION=1.0 bundle exec rake geocombine:index
+OGM_PATH=/path/to/your/files rake geocombine:index
+```
+
+For a single file, it works the same way:
+
+```sh
+bundle exec rake geocombine:index\[/path/to/your/file.json\]
+```
+
+Or:
+
+```sh
+OGM_PATH=/path/to/your/file.json rake geocombine:index
 ```
 
 ### Harvesting and indexing documents from GeoBlacklight sites
@@ -226,12 +213,75 @@ GeoCombine::GeoBlacklightIndexer.document_transformer = -> (document) do
 end
 ```
 
+### Converting metadata
+
+#### Converting metadata into GeoBlacklight JSON
+
+GeoCombine provides several classes representing different metadata standards that implement the `#to_geoblacklight` method for generating records in the [GeoBlacklight JSON format](https://opengeometadata.org/reference/):
+
+```ruby
+GeoCombine::Iso19139 # ISO 19139 XML
+GeoCombine::OGP # OpenGeoPortal JSON
+GeoCombine::Fgdc # FGDC XML
+GeoCombine::EsriOpenData # Esri Open Data Portal JSON
+GeoCombine::CkanMetadata # CKAN JSON
+```
+
+An example for converting an ISO 19139 XML record:
+
+```ruby
+# Create a new ISO19139 object
+> iso_metadata =  GeoCombine::Iso19139.new('./tmp/opengeometadata/edu.stanford.purl/bb/338/jh/0716/iso19139.xml')
+
+# Convert to GeoBlacklight's metadata format
+> iso_metadata.to_geoblacklight
+
+# Output it as JSON instead of a Ruby hash
+> iso_metadata.to_geoblacklight.to_json
+```
+
+Some formats also support conversion into HTML for display in a web browser:
+
+```ruby
+# Create a new ISO19139 object
+> iso_metadata =  GeoCombine::Iso19139.new('./tmp/opengeometadata/edu.stanford.purl/bb/338/jh/0716/iso19139.xml')
+
+# Convert ISO to HTML
+> iso_metadata.to_html
+```
+
+### Migrating metadata
+
+You can use the `GeoCombine::Migrators` to migrate metadata from one schema to another.
+
+Currently, the only migrator is `GeoCombine::Migrators::V1AardvarkMigrator` which migrates from the [GeoBlacklight v1 schema](https://github.com/OpenGeoMetadata/opengeometadata.github.io/blob/main/docs/gbl-1.0.md) to the [Aardvark schema](https://github.com/OpenGeoMetadata/opengeometadata.github.io/blob/main/docs/ogm-aardvark.md)
+
+```ruby
+# Load a record in geoblacklight v1 schema
+record = JSON.parse(File.read('.spec/fixtures/docs/full_geoblacklight.json'))
+
+# Migrate it to Aardvark schema
+GeoCombine::Migrators::V1AardvarkMigrator.new(v1_hash: record).run
+```
+
+Some fields cannot be migrated automatically. To handle the migration of collection names to IDs when migrating from v1 to Aardvark, you can provide a mapping of collection names to IDs to the migrator:
+
+```ruby
+# You can store this mapping as a JSON or CSV file and load it into a hash
+id_map = {
+  'My Collection 1' => 'institution:my-collection-1',
+  'My Collection 2' => 'institution:my-collection-2'
+}
+
+GeoCombine::Migrators::V1AardvarkMigrator.new(v1_hash: record, collection_id_map: id_map).run
+```
+
 ## Tests
 
 To run the tests, use:
 
 ```sh
-$ bundle exec rake spec
+bundle exec rake spec
 ```
 
 ## Contributing

data/geo_combine.gemspec

@@ -32,8 +32,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rspec-html-matchers'
-  spec.add_development_dependency 'rubocop', '~> 1.25'
-  spec.add_development_dependency 'rubocop-rspec', '~> 2.8'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'rubocop-rspec'
   spec.add_development_dependency 'rubocop-rake'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'webmock', '~> 3.14'

data/lib/geo_combine/bounding_box.rb

@@ -48,8 +48,6 @@ module GeoCombine
     end
 
     def self.from_envelope(envelope)
-      return if envelope.nil?
-
       envelope = envelope[/.*ENVELOPE\(([^)]*)/, 1].split(',')
       new(
         west: envelope[0],
@@ -63,8 +61,6 @@ module GeoCombine
     # @param [String] spatial w,s,e,n or w s e n
     # @param [String] delimiter "," or " "
     def self.from_string_delimiter(spatial, delimiter: ',')
-      return if spatial.nil?
-
       spatial = spatial.split(delimiter)
       new(
         west: spatial[0],

data/lib/geo_combine/ckan_metadata.rb

@@ -32,7 +32,7 @@ module GeoCombine
         layer_slug_s: @metadata['name'],
         solr_geom: envelope,
         dc_subject_sm: subjects,
-        dct_references_s: external_references.to_json.to_s,
+        dct_references_s: external_references.to_json,
         dc_format_s: downloadable? ? 'ZIP' : nil # TODO: we only allow direct ZIP file downloads
       }.compact
     end

data/lib/geo_combine/esri_open_data.rb

@@ -4,6 +4,7 @@ module GeoCombine
   # Data model for ESRI's open data portal metadata
   class EsriOpenData
     include GeoCombine::Formatting
+
     attr_reader :metadata
 
     ##

data/lib/geo_combine/geoblacklight.rb

@@ -42,6 +42,7 @@ module GeoCombine
       metadata.each do |key, value|
         translate_formats(key, value)
         enhance_subjects(key, value)
+        infer_geometry_type_from_subject(key, value)
         format_proper_date(key, value)
         fields_should_be_array(key, value)
         translate_geometry_type(key, value)
@@ -55,20 +56,29 @@ module GeoCombine
       metadata.to_json(options)
     end
 
+    # True if the GeoBlacklight-Schema document is valid
+    # @return [Boolean]
+    def valid?
+      validate!
+      true
+    rescue StandardError
+      false
+    end
+
     ##
     # Validates a GeoBlacklight-Schema json document
     # @return [Boolean]
-    def valid?
-      JSON::Validator.validate!(schema, to_json, fragment: '#/definitions/layer') &&
-        dct_references_validate! &&
-        spatial_validate!
+    def validate!
+      JSON::Validator.validate!(schema, to_json, fragment: '#/definitions/layer')
+      validate_references!
+      validate_spatial!
     end
 
     ##
     # Validate dct_references_s
     # @return [Boolean]
-    def dct_references_validate!
-      return true unless metadata.key?('dct_references_s') # TODO: shouldn't we require this field?
+    def validate_references!
+      return unless metadata.key?('dct_references_s') # Not required
 
       begin
         ref = JSON.parse(metadata['dct_references_s'])
@@ -83,8 +93,8 @@ module GeoCombine
       end
     end
 
-    def spatial_validate!
-      GeoCombine::BoundingBox.from_envelope(metadata['solr_geom']).valid?
+    def validate_spatial!
+      raise GeoCombine::Exceptions::InvalidGeometry unless GeoCombine::BoundingBox.from_envelope(metadata['solr_geom']).valid?
     end
 
     private
@@ -106,6 +116,17 @@ module GeoCombine
       metadata[key] = geometry_types[value]
     end
 
+    ##
+    # Enhances empty 'layer_geom_type_s' field by populating from related subject
+    def infer_geometry_type_from_subject(key, value)
+      return unless key == 'layer_geom_type_s' && value.blank?
+
+      # If any subjects match a known geometry type, use the first that does as the geometry type
+      metadata['dc_subject_sm'].find { |subject| geometry_types.value?(subject.capitalize) }&.tap do |subject|
+        metadata[key] = subject.capitalize
+      end
+    end
+
     ##
     # Enhances the 'dc_subject_sm' field by translating subjects to ISO topic
     # categories
@@ -163,7 +184,7 @@ module GeoCombine
       metadata.except!(*DEPRECATED_KEYS_V1)
 
       # ensure we have a proper v1 record
-      valid?
+      validate!
     end
 
     def schema

data/lib/geo_combine/harvester.rb

@@ -11,35 +11,33 @@ module GeoCombine
   class Harvester
     attr_reader :ogm_path, :schema_version
 
-    # Non-metadata repositories that shouldn't be harvested
-    def self.denylist
-      [
-        'GeoCombine',
-        'aardvark',
-        'metadata-issues',
-        'ogm_utils-python',
-        'opengeometadata.github.io',
-        'opengeometadata-rails',
-        'gbl-1_to_aardvark'
-      ]
-    end
-
     # GitHub API endpoint for OpenGeoMetadata repositories
     def self.ogm_api_uri
       URI('https://api.github.com/orgs/opengeometadata/repos?per_page=1000')
     end
 
+    # Initialize a new harvester
+    # @param ogm_path [String] path to the directory where repositories will be cloned
+    # @param skip_repos [Array<String>] list of repository names to skip
+    # @param skip_restricted [Boolean] whether to skip indexing restricted documents
+    # @param schema_version [String] schema version to filter repositories by
+    # @param logger [Logger] logger to use for logging messages
     def initialize(
-      ogm_path: ENV.fetch('OGM_PATH', 'tmp/opengeometadata'),
+      ogm_path:,
+      skip_repos: ENV.fetch('OGM_SKIP_REPOS', '').split(',').map(&:strip),
+      skip_restricted: ENV.fetch('OGM_SKIP_RESTRICTED', 'true') == 'true',
       schema_version: ENV.fetch('SCHEMA_VERSION', 'Aardvark'),
       logger: GeoCombine::Logger.logger
     )
       @ogm_path = ogm_path
       @schema_version = schema_version
+      @skip_repos = skip_repos
+      @skip_restricted = skip_restricted
       @logger = logger
     end
 
     # Enumerable of docs to index, for passing to an indexer
+    # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
     def docs_to_index
       return to_enum(:docs_to_index) unless block_given?
 
@@ -53,19 +51,34 @@ module GeoCombine
 
         doc = JSON.parse(File.read(path))
         [doc].flatten.each do |record|
-          # skip indexing if this record has a different schema version than what we want
           record_schema = record['gbl_mdVersion_s'] || record['geoblacklight_version']
           record_id = record['layer_slug_s'] || record['dc_identifier_s']
+          record_rights = record['dct_accessRights_s'] || record['dc_rights_s']
+
+          # skip indexing if no identifiable schema version
+          unless record_schema
+            @logger.debug "skipping #{record_id || path}; no schema version declared in record"
+            next
+          end
+
+          # skip indexing if this record has a different schema version than what we want
           if record_schema != @schema_version
             @logger.debug "skipping #{record_id}; schema version #{record_schema} doesn't match #{@schema_version}"
             next
           end
 
+          # skip indexing if this record is restricted and we want to skip restricted records
+          if @skip_restricted && record_rights == 'Restricted'
+            @logger.debug "skipping #{record_id}; access rights are restricted"
+            next
+          end
+
           @logger.debug "found record #{record_id} at #{path}"
           yield record, path
         end
       end
     end
+    # rubocop:enable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
 
     # Update a repository via git
     # If the repository doesn't exist, clone it.
@@ -87,19 +100,20 @@ module GeoCombine
     end
 
     # Clone a repository via git
-    # If the repository already exists, skip it.
+    # Return the name of the repository cloned, or nil if skipped
     def clone(repo)
       repo_path = File.join(@ogm_path, repo)
       repo_info = repository_info(repo)
       repo_url = "https://github.com/OpenGeoMetadata/#{repo}.git"
-
-      # Skip if exists; warn if archived or empty
-      if File.directory? repo_path
-        @logger.warn "skipping clone to #{repo_path}; directory exists"
-        return nil
+      repo_schemas = Array(repo_info.dig('custom_properties', 'supported_schemas'))
+
+      # Skip if exists, archived, empty, or different schema
+      return @logger.warn "skipping clone to #{repo_path}; directory exists" if File.directory? repo_path
+      return @logger.warn "repository is archived: #{repo_url}" if repo_info['archived']
+      return @logger.warn "repository is empty: #{repo_url}" if repo_info['size'].zero?
+      unless repo_schemas.include? @schema_version
+        return @logger.warn "repository #{repo_url} clone to #{repo_path}; repository properties don't include schema version #{@schema_version} (found #{repo_schemas.join(', ')})"
       end
-      @logger.warn "repository is archived: #{repo_url}" if repo_info['archived']
-      @logger.warn "repository is empty: #{repo_url}" if repo_info['size'].zero?
 
       Git.clone(repo_url, nil, path: ogm_path, depth: 1)
       @logger.info "cloned #{repo_url} to #{repo_path}"
@@ -119,12 +133,14 @@ module GeoCombine
     # List of repository names to harvest
     def repositories
       @repositories ||= JSON.parse(Net::HTTP.get(self.class.ogm_api_uri))
+                            .filter { |repo| Array(repo.dig('custom_properties', 'supported_schemas')).include? @schema_version }
                             .filter { |repo| repo['size'].positive? }
                             .reject { |repo| repo['archived'] }
+                            .reject { |repo| @skip_repos.include?(repo['name']) }
                             .map { |repo| repo['name'] }
-                            .reject { |name| self.class.denylist.include? name }
     end
 
+    # Fetch repository metadata from GitHub API
     def repository_info(repo_name)
       JSON.parse(Net::HTTP.get(URI("https://api.github.com/repos/opengeometadata/#{repo_name}")))
     end