geo_combine 0.9.0 → 0.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b168ca81c2b6d5ff2fa0ce75d18154ad9806f5a872cc6f96e41ab0cea628864
-  data.tar.gz: 9ba88c0cca642ebe79301182f992f2dfe223bc1cf4abaa9b00ac4687293b78f4
+  metadata.gz: f3486b76e27b3a26ad35ed97c2ed196890ea7ea0a4f707f8cfebca3967cd1472
+  data.tar.gz: 0c2d1e07f52144bd96faec67b83facb0a379fbc17f69fcaa36fd584da5b32773
 SHA512:
-  metadata.gz: 8c54ead8b591bd20fc3b62fddbfec2b0ac6830972cd85bf49fd31f230c84c8cf1a7bd4c0b56de702a235eca130fc643e4c796cb84adf15fd93f0b73075161fe2
-  data.tar.gz: 28fe9a5209dd77c2f8b60e87fddec8c7892ddf33794afd75522cf09c6827496c445c6c2f52445ad7fa46508ac1a4dcedd0c5068e161d0e176ec7ab232b21d296
+  metadata.gz: 71a0cc0aa25959868ef0eb370d8b53561d3505ee3bdf3c8dbd3d8a1553ff66c1049eed1521ea9ba4e665a615c36712a6dd3d025488ab8e46e1c91b2b0536e4f3
+  data.tar.gz: 95422cfd7054c883a2fad7b31083cd514eb4cecb4ebd978f6df692d35e801484d71b78f4e316a358ad18f67ceb0e13bb1fcb52f6cfee5b10b5c48747294d693b

data/README.md

@@ -32,18 +32,32 @@ $ gem install geo_combine
 ## Usage
 
 ### Converting metadata
-
+#### Converting metadata into GeoBlacklight JSON
+GeoCombine provides several classes representing different metadata standards that implement the `#to_geoblacklight` method for generating records in the [GeoBlacklight JSON format](https://opengeometadata.org/reference/):
+```ruby
+GeoCombine::Iso19139 # ISO 19139 XML
+GeoCombine::OGP # OpenGeoPortal JSON
+GeoCombine::Fgdc # FGDC XML
+GeoCombine::EsriOpenData # Esri Open Data Portal JSON
+GeoCombine::CkanMetadata # CKAN JSON
+```
+An example for converting an ISO 19139 XML record:
 ```ruby
 # Create a new ISO19139 object
 > iso_metadata =  GeoCombine::Iso19139.new('./tmp/opengeometadata/edu.stanford.purl/bb/338/jh/0716/iso19139.xml')
 
-# Convert ISO to GeoBlacklight
+# Convert to GeoBlacklight's metadata format
 > iso_metadata.to_geoblacklight
 
-# Convert that to JSON
+# Output it as JSON instead of a Ruby hash
 > iso_metadata.to_geoblacklight.to_json
+```
+Some formats also support conversion into HTML for display in a web browser:
+```ruby
+# Create a new ISO19139 object
+> iso_metadata =  GeoCombine::Iso19139.new('./tmp/opengeometadata/edu.stanford.purl/bb/338/jh/0716/iso19139.xml')
 
-# Convert ISO (or FGDC) to HTML
+# Convert ISO to HTML
 > iso_metadata.to_html
 ```
 
@@ -73,7 +87,7 @@ id_map = {
 GeoCombine::Migrators::V1AardvarkMigrator.new(v1_hash: record, collection_id_map: id_map).run
 ```
 
-### OpenGeoMetadata
+### Downloading metadata from OpenGeoMetadata
 
 #### Logging
 
@@ -144,6 +158,13 @@ You can also set a the Solr instance URL using `SOLR_URL`:
 $ SOLR_URL=http://www.example.com:1234/solr/collection bundle exec rake geocombine:index
 ```
 
+By default, GeoCombine will index only records using the Aardvark metadata format. If you instead want to index records using an older format (e.g. because your GeoBlacklight instance is version 3 or older), you can set the `SCHEMA_VERSION` environment variable:
+
+```sh
+# Only index schema version 1.0 records
+$ SCHEMA_VERSION=1.0 bundle exec rake geocombine:index
+```
+
 ### Harvesting and indexing documents from GeoBlacklight sites
 
 GeoCombine provides a Harvester class and rake task to harvest and index content from GeoBlacklight sites (or any site that follows the Blacklight API format). Given that the configurations can change from consumer to consumer and site to site, the class provides a relatively simple configuration API. This can be configured in an initializer, a wrapping rake task, or any other ruby context where the rake task our class would be invoked.

data/lib/geo_combine/harvester.rb

@@ -31,7 +31,7 @@ module GeoCombine
 
     def initialize(
       ogm_path: ENV.fetch('OGM_PATH', 'tmp/opengeometadata'),
-      schema_version: ENV.fetch('SCHEMA_VERSION', '1.0'),
+      schema_version: ENV.fetch('SCHEMA_VERSION', 'Aardvark'),
       logger: GeoCombine::Logger.logger
     )
       @ogm_path = ogm_path

data/lib/geo_combine/indexer.rb

@@ -35,16 +35,10 @@ module GeoCombine
       start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
       # Index in batches; set batch size via BATCH_SIZE
-      batch = []
-      docs.each do |doc, path|
-        if batch.size < @batch_size
-          batch << [doc, path]
-        else
-          total_indexed += index_batch(batch)
-          batch = []
-        end
+      docs.each_slice(@batch_size) do |slice|
+        batch = slice.map { |doc, path| [doc, path] }
+        total_indexed += index_batch(batch)
       end
-      total_indexed += index_batch(batch) unless batch.empty?
 
       # Issue a commit to make sure all documents are indexed
       @solr.commit

data/lib/geo_combine/migrators/v1_aardvark_migrator.rb

@@ -50,6 +50,7 @@ module GeoCombine
       end
 
       # Convert non-crosswalked fields via lookup tables
+      # rubocop:disable Metrics/PerceivedComplexity
       def convert_non_crosswalked_fields
         # Keys may or may not include whitespace, so we normalize them.
         # Resource class is required so we default to "Other"; resource type is not required.
@@ -57,6 +58,9 @@ module GeoCombine
         resource_type = RESOURCE_TYPE_MAP[@v1_hash['layer_geom_type_s']&.gsub(/\s+/, '')]
         @v2_hash['gbl_resourceType_sm'] = resource_type unless resource_type.nil?
 
+        # If locn_geometry is in the ENVELOPE format, also add it as dcat_bbox
+        @v2_hash['dcat_bbox'] = @v2_hash['locn_geometry'] if @v2_hash['locn_geometry']&.match?(/ENVELOPE/)
+
         # If the user specified a collection id map, use it to convert the collection names to ids
         is_part_of = @v1_hash['dct_isPartOf_sm']&.map { |name| @collection_id_map[name] }&.compact
         if is_part_of.present?
@@ -65,6 +69,7 @@ module GeoCombine
           @v2_hash.delete('dct_isPartOf_sm')
         end
       end
+      # rubocop:enable Metrics/PerceivedComplexity
 
       # Remove fields that are no longer used
       def remove_deprecated_fields
@@ -80,7 +85,7 @@ module GeoCombine
         'dc_publisher_s' => 'dct_publisher_sm', # new namespace; single to multi-valued
         'dct_provenance_s' => 'schema_provider_s', # new URI name
         'dc_subject_sm' => 'dct_subject_sm', # new namespace
-        'solr_geom' => 'dcat_bbox', # new URI name
+        'solr_geom' => 'locn_geometry', # new URI name
         'solr_year_i' => 'gbl_indexYear_im', # new URI name; single to multi-valued
         'dc_source_sm' => 'dct_source_sm', # new namespace
         'dc_rights_s' => 'dct_accessRights_s', # new URI name

data/lib/geo_combine/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GeoCombine
-  VERSION = '0.9.0'
+  VERSION = '0.9.2'
 end

data/spec/fixtures/docs/full_geoblacklight_aardvark.json

@@ -44,6 +44,7 @@
     "stanford-rb371kw9607"
   ],
   "dcat_bbox":"ENVELOPE(29.572742, 35.000308, 4.234077, -1.478794)",
+  "locn_geometry":"ENVELOPE(29.572742, 35.000308, 4.234077, -1.478794)",
   "gbl_indexYear_im":[
     2005
   ],

data/spec/lib/geo_combine/harvester_spec.rb

@@ -5,7 +5,7 @@ require 'geo_combine/harvester'
 require 'spec_helper'
 
 RSpec.describe GeoCombine::Harvester do
-  subject(:harvester) { described_class.new(ogm_path: 'spec/fixtures/indexing', logger:) }
+  subject(:harvester) { described_class.new(ogm_path: 'spec/fixtures/indexing', schema_version: '1.0') }
 
   let(:logger) { instance_double(Logger, warn: nil, info: nil, error: nil, debug: nil) }
   let(:repo_name) { 'my-institution' }

data/spec/lib/geo_combine/indexer_spec.rb

@@ -96,6 +96,18 @@ RSpec.describe GeoCombine::Indexer do
       )
     end
 
+    context 'when the number of docs is greater than batch size' do
+      before do
+        stub_const('ENV', 'SOLR_BATCH_SIZE' => 10)
+      end
+
+      let(:docs) { (1..40).map { |n| [{ 'id' => n }, "path/to/record#{n}.json"] } }
+
+      it 'indexes the correct number of documents' do
+        expect(indexer.index(docs)).to eq 40
+      end
+    end
+
     it 'commits changes to solr after indexing' do
       indexer.index(docs)
       expect(solr).to have_received(:commit).once

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: geo_combine
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.9.2
 platform: ruby
 authors:
 - Jack Reed
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-29 00:00:00.000000000 Z
+date: 2024-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -376,7 +376,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: A Ruby toolkit for managing geospatial metadata