geo_combine 0.11.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 21614ec82bcbe5c5a7bb1291e398795f1baff048552746335f18f315da677c9e
-  data.tar.gz: 6c94a18a0f43fd141f5967930a909efac60ab560ac91a0759e48cc45fdab39b5
+  metadata.gz: d06fb38092cd8d8db20f1c9dfa14c82c19162d23415d1c34d92a2edf3826ff56
+  data.tar.gz: c2f2e35183978ec29e04d976438c9c5c567dc05341b128074366359d6164b027
 SHA512:
-  metadata.gz: 75ce7b76dfa6ad5905410e6b38a18ee98185c50d5d116c8fc5794eba3376e85a05d2e2ad7d70d6bde8de7b0073df22f15cb28759633e0d3c6e546e24a13dee82
-  data.tar.gz: cbd5ccc9eac444e599e0dce59fda1f23ef752cec71508a83fbe32ea1b2785b5a8c52d66c4ebcebc81a913aaccf6f677f5555622a43a8e16e902ab5f3379cbfa0
+  metadata.gz: 13720c277a1d399d83a882df36eaad9f6c5cbef537fd2753f8391cc8ec73ab9f524308b2adc9c2b1b3bcefff44b928bd4d6b3fa5d05973b492e90bf79c0d524e
+  data.tar.gz: b9a42c999193d944a4c0b9fb309bcbb27a0f66bd8a022ea894cf7594acaf4370d66d1ed2a6d8b7601a1f8b6af1965dc343b823ebcf920790e5fe5e2a14de748d

data/.github/workflows/ruby.yml

@@ -6,11 +6,11 @@ jobs:
   rubocop:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
     - name: Set up Ruby and install dependencies
       uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 3.2
+        ruby-version: 3.3
         bundler-cache: true
     - name: Run linter
       run: bundle exec rubocop
@@ -18,27 +18,50 @@ jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        ruby: [3.2, 3.3, 3.4]
-        faraday_version: ['', '~> 1.0'] # Defaults to whatever's the most recent version.
+        ruby: [3.3, 3.4, 4.0]
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
 
     - name: Set up Ruby and install dependencies
       uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
         bundler-cache: true
-      env:
-        FARADAY_VERSION: ${{ matrix.faraday_version }}
 
-    - name: "Run tests for Ruby ${{ matrix.ruby }} and Faraday ${{ matrix.faraday_version == '~> 1.0' && '1.x' || 'latest' }}"
+    - name: "Run tests for Ruby ${{ matrix.ruby }}"
       run: bundle exec rake spec
-      env:
-        FARADAY_VERSION: ${{ matrix.faraday_version }}
 
     - name: Upload coverage artifacts
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@v7
       with:
-        name: "coverage-${{ matrix.ruby }}-${{ matrix.faraday_version == '~> 1.0' && 'faraday-1.x' || 'faraday-latest' }}"
+        name: "coverage-${{ matrix.ruby }}"
         path: coverage/
+
+  # Runs the specs that need a real Solr, using the GeoBlacklight schema
+  integration:
+    runs-on: ubuntu-latest
+    env:
+      GEOBLACKLIGHT_VERSION: v5.4.0
+      SOLR_INTEGRATION_URL: http://localhost:8983/solr/blacklight-core
+    steps:
+    - uses: actions/checkout@v7
+
+    - name: Start Solr with the GeoBlacklight ${{ env.GEOBLACKLIGHT_VERSION }} configuration
+      run: |
+        mkdir -p tmp/solr-conf
+        curl -sfL "https://github.com/geoblacklight/geoblacklight/archive/refs/tags/${GEOBLACKLIGHT_VERSION}.tar.gz" \
+          | tar -xz -C tmp/solr-conf --strip-components=3 "geoblacklight-${GEOBLACKLIGHT_VERSION#v}/solr/conf"
+        rm tmp/solr-conf/core.properties
+        docker run -d -p 8983:8983 -v "$PWD/tmp/solr-conf:/gbl-conf:ro" solr:9 solr-precreate blacklight-core /gbl-conf
+        timeout 120 sh -c 'until curl -sf "$SOLR_INTEGRATION_URL/get?id=ready" > /dev/null; do sleep 2; done'
+
+    - name: Set up Ruby and install dependencies
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 3.4
+        bundler-cache: true
+
+    - name: Run tests, including integration tests against Solr
+      run: bundle exec rake spec

data/.rubocop.yml

@@ -24,3 +24,7 @@ RSpec/MultipleMemoizedHelpers:
 RSpec/BeforeAfterAll:
   Exclude:
     - 'spec/lib/tasks/geo_combine_spec.rb'
+
+Metrics/BlockLength:
+  Exclude:
+    - 'lib/tasks/*.rake'

data/.rubocop_todo.yml

@@ -64,7 +64,6 @@ RSpec/ExampleLength:
 # Offense count: 4
 RSpec/ExpectInHook:
   Exclude:
-    - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
     - 'spec/lib/geo_combine/geoblacklight_spec.rb'
 
 # Offense count: 23
@@ -73,7 +72,6 @@ RSpec/ExpectInHook:
 RSpec/MessageSpies:
   Exclude:
     - 'spec/lib/geo_combine/esri_open_data_spec.rb'
-    - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
     - 'spec/lib/geo_combine/geoblacklight_spec.rb'
     - 'spec/lib/geo_combine/ogp_spec.rb'
     - 'spec/lib/geo_combine_spec.rb'
@@ -121,7 +119,6 @@ RSpec/SpecFilePathFormat:
 RSpec/StubbedMock:
   Exclude:
     - 'spec/lib/geo_combine/esri_open_data_spec.rb'
-    - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
     - 'spec/lib/geo_combine/geoblacklight_spec.rb'
     - 'spec/lib/geo_combine/ogp_spec.rb'
     - 'spec/lib/geo_combine_spec.rb'
@@ -131,16 +128,11 @@ RSpec/SubjectStub:
   Exclude:
     - 'spec/lib/geo_combine/ogp_spec.rb'
 
-# Offense count: 1
-# Configuration parameters: IgnoreNameless, IgnoreSymbolicNames.
-RSpec/VerifiedDoubles:
-  Exclude:
-    - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
-
 # Offense count: 1
 Security/Open:
   Exclude:
     - 'lib/geo_combine/geoblacklight.rb'
+    - 'lib/geo_combine/geoblacklight_aardvark.rb'
 
 # Offense count: 6
 # Configuration parameters: AllowedConstants.
@@ -150,8 +142,8 @@ Style/Documentation:
     - 'test/**/*'
     - 'lib/geo_combine/bounding_box.rb'
     - 'lib/geo_combine/ckan_metadata.rb'
-    - 'lib/geo_combine/geo_blacklight_harvester.rb'
     - 'lib/geo_combine/geoblacklight.rb'
+    - 'lib/geo_combine/geoblacklight_aardvark.rb'
     - 'lib/geo_combine/geometry_types.rb'
     - 'lib/geo_combine/iso19139.rb'
 

data/Gemfile

@@ -7,3 +7,6 @@ gemspec
 
 gem 'byebug', require: false
 gem 'rdbg', require: false
+
+# Json 3 drops quicks_mode which we use in our json-schema
+gem 'json', '< 3'

data/README.md

@@ -1,14 +1,14 @@
 # GeoCombine
 
 ![CI](https://github.com/OpenGeoMetadata/GeoCombine/actions/workflows/ruby.yml/badge.svg)
-[![Coverage Status](https://img.shields.io/badge/coverage-95%25-brightgreen)]()
+[![Coverage Status](https://img.shields.io/badge/coverage-100%25-brightgreen)]()
 [![Gem Version](https://img.shields.io/gem/v/geo_combine.svg)](https://github.com/OpenGeoMetadata/GeoCombine/releases)
 
 A Ruby toolkit for managing geospatial metadata, including:
 
 - harvesting and indexing records from OpenGeoMetadata
-- harvesting records directly from a GeoBlacklight instance
 - converting geospatial metadata records between standards
+- migrating OpenGeoMetadata records from older to newer schema versions
 
 ## Installation
 
@@ -54,6 +54,14 @@ By default, GeoCombine will only fetch and index records that are not marked as
 export OGM_SKIP_RESTRICTED=false
 ```
 
+#### GitHub API token
+
+GeoCombine uses the GitHub API to find OpenGeoMetadata repositories. Unauthenticated requests are limited to 60 per hour, and `geocombine:clone` makes one for each repository. If you hit the limit, set the `GITHUB_TOKEN` environment variable to a GitHub personal access token to raise it; the token only needs read access to public repositories:
+
+```sh
+export GITHUB_TOKEN=your-token
+```
+
 #### Clone OpenGeoMetadata repositories locally
 
 ```sh
@@ -118,6 +126,36 @@ You can also set the Solr instance URL using `SOLR_URL`:
 SOLR_URL=http://www.example.com:1234/solr/collection bundle exec rake geocombine:index
 ```
 
+#### Withdrawn records
+
+An OpenGeoMetadata repository can withdraw records by listing them in a `withdrawn.json` file at its root:
+
+```json
+{
+  "$schema": "https://opengeometadata.org/schema/ogm-withdrawals-1.0.json",
+  "ogm_version": "1.0",
+  "withdrawn": [
+    { "id": "stanford-bb058zh0946", "date": "2026-03-14", "reason": "superseded" }
+  ]
+}
+```
+
+After indexing, `geocombine:index` deletes the records these files list from your Solr index. It is careful about what it deletes:
+
+- A record is only deleted if a `withdrawn.json` lists its `id` (or `layer_slug_s`, for `SCHEMA_VERSION=1.0`). Removing a record's file from a repository does not delete it from your index.
+- A record that is still present in the repository is never deleted, even if it is also listed as withdrawn. GeoCombine logs a warning instead.
+- A `withdrawn.json` that can't be read is logged and skipped; it doesn't stop indexing.
+- Repositories listed in `OGM_SKIP_REPOS` can't delete anything.
+- If the harvest stops before it has walked every record, nothing is deleted.
+
+GeoCombine keeps no record of what it has deleted. Every run deletes every withdrawn record again, which changes nothing for records that are already gone. Deletes match by `id` alone, so a withdrawn `id` that matches a record you indexed from somewhere other than OpenGeoMetadata will delete that record too.
+
+To delete withdrawn records without reindexing everything, run:
+
+```sh
+bundle exec rake geocombine:withdraw
+```
+
 #### Logging
 
 Some of the tools and scripts in this gem use Ruby's `Logger` class to print information to `$stderr`. By default, the log level is set to `Logger::INFO`. For more verbose information, you can set the `LOG_LEVEL` environment variable to `DEBUG`:
@@ -152,67 +190,6 @@ Or:
 OGM_PATH=/path/to/your/file.json rake geocombine:index
 ```
 
-### Harvesting and indexing documents from GeoBlacklight sites
-
-GeoCombine provides a Harvester class and rake task to harvest and index content from GeoBlacklight sites (or any site that follows the Blacklight API format). Given that the configurations can change from consumer to consumer and site to site, the class provides a relatively simple configuration API. This can be configured in an initializer, a wrapping rake task, or any other ruby context where the rake task our class would be invoked.
-
-```sh
-bundle exec rake geocombine:geoblacklight_harvester:index[YOUR_CONFIGURED_SITE_KEY]
-```
-
-#### Harvester configuration
-
-Only the sites themselves are required to be configured but there are various configuration options that can (optionally) be supplied to modify the harvester's behavior.
-
-```ruby
-GeoCombine::GeoBlacklightHarvester.configure do
-  {
-    commit_within: '10000',
-    crawl_delay: 1, # All sites
-    debug: true,
-    SITE1: {
-      crawl_delay: 2, # SITE1 only
-      host: 'https://geoblacklight.example.edu',
-      params: {
-        f: {
-          dct_provenance_s: ['Institution']
-        }
-      }
-    },
-    SITE2: {
-      host: 'https://geoportal.example.edu',
-      params: {
-        q: '*'
-      }
-    }
-  }
-end
-```
-
-##### Crawl Delays (default: none)
-
-Crawl delays can be configured (in seconds) either globally for all sites or on a per-site basis. This will cause a delay for that number of seconds between each search results page (note that Blacklight 7 necessitates a lot of requests per results page and this only causes the delay per page of results)
-
-##### Solr's commitWithin (default: 5000 milliseconds)
-
-Solr's commitWithin option can be configured (in milliseconds) by passing a value under the commit_within key.
-
-#### Transforming Documents
-
-You may need to transform documents that are harvested for various purposes (removing fields, adding fields, omitting a document all together, etc). You can configure some ruby code (a proc) that will take the document in, transform it, and return the transformed document. By default the indexer will remove the `score`, `timestamp`, and `_version_` fields from the documents harvested. If you provide your own transformer, you'll likely want to remove these fields in addition to the other transformations you provide.
-
-```ruby
-GeoCombine::GeoBlacklightIndexer.document_transformer = -> (document) do
-  # Removes "bogus_field" from the content we're harvesting
-  # in addition to some other solr fields we don't want
-  %w[_version_ score timestamp bogus_field].each do |field|
-    document.delete(field)
-  end
-
-  document
-end
-```
-
 ### Converting metadata
 
 #### Converting metadata into GeoBlacklight JSON
@@ -250,6 +227,66 @@ Some formats also support conversion into HTML for display in a web browser:
 > iso_metadata.to_html
 ```
 
+#### Converting metadata into Aardvark JSON
+
+`GeoCombine::Fgdc` and `GeoCombine::Iso19139` also implement `#to_aardvark`, which converts FGDC or ISO 19139 XML into the [OGM Aardvark schema](https://opengeometadata.org/ogm-aardvark/) and returns a `GeoCombine::GeoblacklightAardvark`:
+
+```ruby
+# Create a new Fgdc object
+> fgdc_metadata = GeoCombine::Fgdc.new('./tmp/opengeometadata/edu.tufts/0/108/220/208/fgdc.xml')
+
+# Convert to the Aardvark schema
+> aardvark = fgdc_metadata.to_aardvark
+
+# Validate it against the Aardvark JSON schema
+> aardvark.valid?
+=> true
+
+# Output it as JSON instead of a Ruby hash
+> aardvark.to_json
+```
+
+Aardvark defines some fields that FGDC and ISO have no source for, such as `gbl_suppressed_b`, and the links in a record may not be the ones you want to publish. Pass fields in and they are merged into the transformed record, replacing any derived values:
+
+```ruby
+> fgdc_metadata.to_aardvark(
+    'id' => 'tufts-ecuador-drilling-towers',
+    'schema_provider_s' => 'Tufts',
+    'dct_references_s' => { 'http://schema.org/url' => 'https://example.edu/catalog/tufts-1' }.to_json
+  )
+```
+
+Note about `schema_provider_s` and `id`:
+
+```ruby
+# Without a provider, id and provider are derived from the record
+> fgdc_metadata.to_aardvark.metadata['id']
+=> "tufts-university-gis-center-ecuador50kdrillingtower11"
+
+# With one, the provider is used as the id prefix
+> fgdc_metadata.to_aardvark('schema_provider_s' => 'Tufts').metadata['id']
+=> "tufts-ecuador50kdrillingtower11"
+
+# Or set the id yourself
+> fgdc_metadata.to_aardvark('id' => 'tufts-ecuador-drilling-towers').metadata['id']
+=> "tufts-ecuador-drilling-towers"
+```
+
+The FGDC transformation fills in `dct_references_s` from the citation's online linkage and the network resources of each digital form, classified by URL and by the format they're distributed in: a landing page, downloads (a labeled list when there's more than one), FGDC and HTML metadata, and OGC (WMS, WFS, WCS, WMTS) and Esri service endpoints. A browse graphic URL becomes the thumbnail, and `gbl_wxsIdentifier_s` is the layer an OGC request asks for. References you pass in replace the derived ones.
+
+ISO 19139 records (including ISO 19115-2 `gmi:MI_Metadata`) work the same way. Their ids use the last segment of the `fileIdentifier`, falling back to the `dataSetURI`, the citation identifier and the title:
+
+```ruby
+> iso_metadata = GeoCombine::Iso19139.new('./tmp/opengeometadata/edu.stanford.purl/bb/338/jh/0716/iso19139.xml')
+
+> iso_metadata.to_aardvark('schema_provider_s' => 'Stanford').metadata['id']
+=> "stanford-bb338jh0716"
+```
+
+The ISO transformation also fills in `dct_references_s`, from the dataset URI, downloads, and OGC (WMS, WFS, WCS, WMTS) and Esri service links, along with `gbl_wxsIdentifier_s` from the name of an OGC link. References you pass in replace the derived ones.
+
+Some values can't be derived reliably and are left for you to pass in: `gbl_resourceType_sm` for vector data whose geometry is only described as `composite` (used for both lines and polygons), and `pcdm_memberOf_sm` and `dct_isPartOf_sm`, which take the ids of other Aardvark records.
+
 ### Migrating metadata
 
 You can use the `GeoCombine::Migrators` to migrate metadata from one schema to another.
@@ -284,6 +321,12 @@ To run the tests, use:
 bundle exec rake spec
 ```
 
+Some tests need a real Solr core running the GeoBlacklight schema. They delete documents, so they only run when `SOLR_INTEGRATION_URL` points at a core you don't mind them changing:
+
+```sh
+SOLR_INTEGRATION_URL=http://localhost:8983/solr/blacklight-core bundle exec rake spec
+```
+
 ## Contributing
 
 1. Fork it ( https://github.com/[my-github-username]/GeoCombine/fork )

data/geo_combine.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'sanitize'
   spec.add_dependency 'thor'
   spec.add_dependency 'faraday-net_http_persistent', '~> 2.0'
-  spec.add_dependency 'git'
+  spec.add_dependency 'git', '~> 6.0'
   spec.add_dependency 'faraday-retry', '~> 2.2'
 
   spec.add_development_dependency 'bundler'

data/lib/geo_combine/exceptions.rb

@@ -7,5 +7,8 @@ module GeoCombine
 
     class InvalidGeometry < StandardError
     end
+
+    class InvalidSchemaVersion < StandardError
+    end
   end
 end

data/lib/geo_combine/fgdc.rb

@@ -11,6 +11,13 @@ module GeoCombine
       Nokogiri::XSLT(File.open(File.join(File.dirname(__FILE__), '../xslt/fgdc2geoBL.xsl')))
     end
 
+    ##
+    # Returns a Nokogiri::XSLT object containing the FGDC to Aardvark XSL
+    # @return [Nokogiri::XSLT]
+    def xsl_aardvark
+      Nokogiri::XSLT(File.open(File.join(File.dirname(__FILE__), '../xslt/fgdc2Aardvark.xsl')))
+    end
+
     ##
     # Returns a Nokogiri::XSLT object containing the ISO19139 to HTML XSL
     # @return [Nokogiri:XSLT]

data/lib/geo_combine/geoblacklight_aardvark.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'open-uri'
+
+module GeoCombine
+  class GeoblacklightAardvark
+    attr_reader :metadata
+
+    SCHEMA_VERSION = 'Aardvark'
+    SCHEMA_JSON_URL = 'https://raw.githubusercontent.com/OpenGeoMetadata/opengeometadata.github.io/main/docs/schema/geoblacklight-schema-aardvark.json'
+    SCHEMA_DRAFT_URL = 'http://json-schema.org/draft-04/schema#'
+
+    ##
+    # @param [String] metadata a JSON string document in the Aardvark schema
+    # @param [Hash] fields values merged with the parsed document, used for
+    # fields Aardvark defines but the source metadata cannot supply
+    def initialize(metadata, fields = {})
+      @metadata = JSON.parse(metadata).merge(fields)
+    end
+
+    ##
+    # Returns a string of JSON from an Aardvark hash
+    # @return [String]
+    def to_json(options = {})
+      metadata.to_json(options)
+    end
+
+    ##
+    # True if the document is a valid Aardvark record
+    # @return [Boolean]
+    def valid?
+      validate!
+      true
+    rescue StandardError
+      false
+    end
+
+    ##
+    # Validates an Aardvark json document
+    def validate!
+      JSON::Validator.validate!(schema, to_json)
+      validate_version!
+      validate_spatial!
+    end
+
+    ##
+    # Validate gbl_mdVersion_s
+    def validate_version!
+      return if metadata['gbl_mdVersion_s'] == SCHEMA_VERSION
+
+      raise GeoCombine::Exceptions::InvalidSchemaVersion,
+            "gbl_mdVersion_s must be #{SCHEMA_VERSION.inspect}, got #{metadata['gbl_mdVersion_s'].inspect}"
+    end
+
+    ##
+    # Validate locn_geometry
+    def validate_spatial!
+      geometry = metadata['locn_geometry']
+      return unless geometry =~ /ENVELOPE\(/
+
+      raise GeoCombine::Exceptions::InvalidGeometry unless GeoCombine::BoundingBox.from_envelope(geometry).valid?
+    end
+
+    private
+
+    def schema
+      @schema ||= JSON.parse(URI.open(SCHEMA_JSON_URL).read).merge('$schema' => SCHEMA_DRAFT_URL)
+    end
+  end
+end