geo_combine 0.9.2 → 0.11.0

data/spec/features/fgdc2html_spec.rb

@@ -5,6 +5,7 @@ require 'spec_helper'
 # TODO: Provide additional expectations on html structure
 describe 'FGDC to html' do
   include XmlDocs
+
   let(:page) { GeoCombine::Fgdc.new(tufts_fgdc).to_html }
 
   describe 'Identification Information' do

data/spec/features/iso2html_spec.rb

@@ -5,6 +5,7 @@ require 'spec_helper'
 # TODO: Provide additional expectations on html structure
 describe 'ISO 19139 to html' do
   include XmlDocs
+
   let(:page) { GeoCombine::Iso19139.new(stanford_iso).to_html }
 
   describe 'Identification Information' do

data/spec/fixtures/indexing/restricted_aardvark.json

@@ -0,0 +1,34 @@
+{
+  "dc_publisher_s": "",
+  "dc_type_s": "Dataset",
+  "dct_title_s": "Property Lot, Arlington County, VA ",
+  "dct_description_sm": [
+    "This polygon represents property lot in Arlington County, VA"
+  ],
+  "schema_provider_s": "GMU",
+  "gbl_resourceType_sm": [
+    "Polygon"
+  ],
+  "dct_subject_sm": [
+    "Property"
+  ],
+  "gbl_indexYear_im": [
+    1
+  ],
+  "locn_geometry": "ENVELOPE(-77.1735, -77.0305, 38.9349, 38.8268)",
+  "dct_accessRights_s": "Restricted",
+  "dct_format_s": "Shapefile",
+  "gbl_wxsIdentifier_s": ":arlington_2004_lot_poly",
+  "id": "gmu-arlington-2004-lot-poly",
+  "dct_identifier_sm": [
+    "GMU.arlington_2004_lot_poly"
+  ],
+  "gbl_mdVersion_s": "Aardvark",
+  "dct_temporal_sm": [
+    "0001-01-01 01:01:01+00:00"
+  ],
+  "dct_spatial_sm": [
+    "virginia"
+  ],
+  "dct_references_s": "{\"http://www.opengis.net/def/serviceType/ogc/wfs\":\"http://localhost:8080/secure-geoserver/wfs\",\"http://www.opengis.net/def/serviceType/ogc/wms\":\"http://localhost:8080/secure-geoserver/wms\"}"
+}

data/spec/fixtures/indexing/restricted_geoblacklight.json

@@ -0,0 +1,42 @@
+{
+  "dc_creator_sm": [
+    "United States. Defense Mapping Agency"
+  ],
+  "dc_description_s": "This raster layer is an image of a 1:5,000,000 scale aeronautical chart sheet that projects to Lambert Conformal Conic. The National Geospatial-Intelligence Agency and its legacy organizations (Defense Mapping Agency, National Imagery and Mapping Agency) originally published the series, and all sheets are in English. East View Geospatial has completed digitization, georeferencing, and electronic archiving services. To see the map index for this chart series, refer to the Derived Data layer.",
+  "dc_format_s": "WorldImage TIFF",
+  "dc_identifier_s": "http://hdl.handle.net/2451/38709",
+  "dc_language_s": "English",
+  "dc_publisher_s": "East View Cartographic, Inc.",
+  "dc_rights_s": "Restricted",
+  "dc_subject_sm": [
+    "Aeronautical charts"
+  ],
+  "dc_title_s": "Global Navigation and Planning Chart, Sheet 21 W ",
+  "dc_type_s": "Dataset",
+  "dct_isPartOf_sm": [
+    "EVG Open Data Package"
+  ],
+  "dct_issued_s": "5/31/2016",
+  "dct_provenance_s": "NYU",
+  "dct_references_s": "{\"http://schema.org/url\":\"http://hdl.handle.net/2451/38709\",\"http://schema.org/downloadUrl\":\"https://archive.nyu.edu/retrieve/84050/nyu_2451_38709.zip\",\"http://www.opengis.net/def/serviceType/ogc/wfs\":\"https://maps-restricted.geo.nyu.edu/geoserver/sdr/wfs\",\"http://www.opengis.net/def/serviceType/ogc/wms\":\"https://maps-restricted.geo.nyu.edu/geoserver/sdr/wms\"}",
+  "dct_source_sm": [
+    "nyu-2451-38715"
+  ],
+  "dct_spatial_sm": [
+
+  ],
+  "dct_temporal_sm": [
+    "2016"
+  ],
+  "geoblacklight_version": "1.0",
+  "layer_geom_type_s": "Raster",
+  "layer_id_s": "sdr:nyu_2451_38709",
+  "layer_modified_dt": "2018-01-16T15:11:26Z",
+  "layer_slug_s": "nyu-2451-38709",
+  "nyu_addl_dspace_s": "39708",
+  "nyu_addl_format_sm": [
+    "WorldImage TIFF"
+  ],
+  "solr_geom": "ENVELOPE(-179.99999999999994, -177.98915207217718, -39.99974999999998, -75.0037465617225)",
+  "solr_year_i": 2016
+}

data/spec/lib/geo_combine/ckan_metadata_spec.rb

@@ -4,6 +4,7 @@ require 'spec_helper'
 
 RSpec.describe GeoCombine::CkanMetadata do
   include JsonDocs
+
   let(:ckan_sample) { described_class.new(ckan_metadata) }
 
   describe '#to_geoblacklight' do

data/spec/lib/geo_combine/esri_open_data_spec.rb

@@ -4,6 +4,7 @@ require 'spec_helper'
 
 RSpec.describe GeoCombine::EsriOpenData do
   include JsonDocs
+
   let(:esri_sample) { described_class.new(esri_opendata_metadata) }
   let(:metadata) { esri_sample.instance_variable_get(:@metadata) }
 

data/spec/lib/geo_combine/fgdc_spec.rb

@@ -4,6 +4,7 @@ require 'spec_helper'
 
 RSpec.describe GeoCombine::Fgdc do
   include XmlDocs
+
   let(:fgdc_object) { described_class.new(tufts_fgdc) }
 
   describe '#initialize' do
@@ -32,7 +33,7 @@ RSpec.describe GeoCombine::Fgdc do
     end
 
     it 'is not valid due to bad modification date but valid otherwise' do
-      expect { fgdc_geobl.valid? }.to raise_error(JSON::Schema::ValidationError, /layer_modified_dt/)
+      expect { fgdc_geobl.validate! }.to raise_error(JSON::Schema::ValidationError, /layer_modified_dt/)
       fgdc_geobl.metadata.delete 'layer_modified_dt'
       expect(fgdc_geobl).to be_valid
     end
@@ -51,7 +52,7 @@ RSpec.describe GeoCombine::Fgdc do
       end
 
       it 'dc_description_s' do
-        expect(fgdc_geobl.metadata['dc_description_s']).to match(/Ecuador created from/)
+        expect(fgdc_geobl.metadata['dc_description_s']).to match('Ecuador created from')
       end
 
       it 'dc_rights_s' do

data/spec/lib/geo_combine/geoblacklight_spec.rb

@@ -6,6 +6,7 @@ RSpec.describe GeoCombine::Geoblacklight do
   include XmlDocs
   include JsonDocs
   include GeoCombine::Exceptions
+
   let(:full_geobl) { described_class.new(full_geoblacklight) }
   let(:enhanced_geobl) { described_class.new(basic_geoblacklight, 'layer_geom_type_s' => 'esriGeometryPolygon') }
   let(:basic_geobl) { described_class.new(basic_geoblacklight) }
@@ -60,11 +61,17 @@ RSpec.describe GeoCombine::Geoblacklight do
     it 'formats the geometry type field' do
       expect(enhanced_geobl.metadata['layer_geom_type_s']).to eq 'Polygon'
     end
+
+    it 'populates the geometry type field from the subject field if not set' do
+      enhanced_from_subject_geobl = described_class.new(basic_geoblacklight, 'dc_subject_sm' => ['polygon'])
+      enhanced_from_subject_geobl.enhance_metadata
+      expect(enhanced_from_subject_geobl.metadata['layer_geom_type_s']).to eq 'Polygon'
+    end
   end
 
-  describe '#valid?' do
+  describe '#validate!' do
     it 'a valid geoblacklight-schema document should be valid' do
-      expect(full_geobl.valid?).to be true
+      expect(full_geobl).to be_valid
     end
 
     context 'must have required fields' do
@@ -78,7 +85,7 @@ RSpec.describe GeoCombine::Geoblacklight do
       ].each do |field|
         it field do
           full_geobl.metadata.delete field
-          expect { full_geobl.valid? }.to raise_error(JSON::Schema::ValidationError, /#{field}/)
+          expect { full_geobl.validate! }.to raise_error(JSON::Schema::ValidationError, /#{field}/)
         end
       end
     end
@@ -104,7 +111,7 @@ RSpec.describe GeoCombine::Geoblacklight do
       ].each do |field|
         it field do
           full_geobl.metadata.delete field
-          expect { full_geobl.valid? }.not_to raise_error
+          expect { full_geobl.validate! }.not_to raise_error
         end
       end
     end
@@ -119,31 +126,25 @@ RSpec.describe GeoCombine::Geoblacklight do
       ].each do |field|
         it field do
           full_geobl.metadata.delete field
-          expect { full_geobl.valid? }.not_to raise_error
+          expect { full_geobl.validate! }.not_to raise_error
         end
       end
     end
 
     it 'an invalid document' do
-      expect { basic_geobl.valid? }.to raise_error JSON::Schema::ValidationError
+      expect { basic_geobl.validate! }.to raise_error JSON::Schema::ValidationError
     end
 
     it 'calls the dct_references_s validator' do
-      expect(enhanced_geobl).to receive(:dct_references_validate!)
+      expect(enhanced_geobl).to receive(:validate_references!)
       enhanced_geobl.valid?
     end
-
-    it 'validates spatial bounding box' do
-      expect(JSON::Validator).to receive(:validate!).and_return true
-      expect { basic_geobl.valid? }
-        .to raise_error GeoCombine::Exceptions::InvalidGeometry
-    end
   end
 
-  describe '#dct_references_validate!' do
+  describe '#validate_references!' do
     context 'with valid document' do
       it 'is valid' do
-        expect(full_geobl.dct_references_validate!).to be true
+        expect { full_geobl.validate_references! }.not_to raise_error
       end
     end
 
@@ -173,22 +174,22 @@ RSpec.describe GeoCombine::Geoblacklight do
       end
 
       it 'unparseable json' do
-        expect { bad_ref.dct_references_validate! }.to raise_error JSON::ParserError
+        expect { bad_ref.validate_references! }.to raise_error JSON::ParserError
       end
 
       it 'not a hash' do
-        expect { not_hash.dct_references_validate! }.to raise_error GeoCombine::Exceptions::InvalidDCTReferences
+        expect { not_hash.validate_references! }.to raise_error GeoCombine::Exceptions::InvalidDCTReferences
       end
     end
   end
 
-  describe 'spatial_validate!' do
+  describe 'validate_spatial!' do
     context 'when valid' do
-      it { expect { full_geobl.spatial_validate! }.not_to raise_error }
+      it { expect { full_geobl.validate_spatial! }.not_to raise_error }
     end
 
     context 'when invalid' do
-      it { expect { basic_geobl.spatial_validate! }.to raise_error GeoCombine::Exceptions::InvalidGeometry }
+      it { expect { basic_geobl.validate_spatial! }.to raise_error GeoCombine::Exceptions::InvalidGeometry }
     end
   end
 

data/spec/lib/geo_combine/harvester_spec.rb

@@ -5,7 +5,7 @@ require 'geo_combine/harvester'
 require 'spec_helper'
 
 RSpec.describe GeoCombine::Harvester do
-  subject(:harvester) { described_class.new(ogm_path: 'spec/fixtures/indexing', schema_version: '1.0') }
+  subject(:harvester) { described_class.new(ogm_path: 'spec/fixtures/indexing', logger: logger) }
 
   let(:logger) { instance_double(Logger, warn: nil, info: nil, error: nil, debug: nil) }
   let(:repo_name) { 'my-institution' }
@@ -14,11 +14,12 @@ RSpec.describe GeoCombine::Harvester do
   let(:stub_repo) { instance_double(Git::Base) }
   let(:stub_gh_api) do
     [
-      { name: repo_name, size: 100 },
-      { name: 'another-institution', size: 100 },
-      { name: 'outdated-institution', size: 100, archived: true }, # archived
-      { name: 'aardvark', size: 300 },                             # on denylist
-      { name: 'empty', size: 0 }                                   # no data
+      { name: repo_name, size: 100, custom_properties: { supported_schemas: ['Aardvark'] } },
+      { name: 'another-institution', size: 100, custom_properties: { supported_schemas: ['Aardvark', '1.0'] } }, # multiple schemas
+      { name: 'v1-institution', size: 300, custom_properties: { supported_schemas: ['1.0'] } }, # schema mismatch
+      { name: 'outdated-institution', size: 100, custom_properties: { supported_schemas: ['Aardvark'] }, archived: true }, # archived
+      { name: 'empty', size: 0, custom_properties: { supported_schemas: ['Aardvark'] } }, # no data
+      { name: 'tool', size: 50 } # not a metadata repository
     ]
   end
 
@@ -42,17 +43,51 @@ RSpec.describe GeoCombine::Harvester do
   describe '#docs_to_index' do
     it 'yields each JSON record with its path, skipping layers.JSON' do
       expect { |b| harvester.docs_to_index(&b) }.to yield_successive_args(
-        [JSON.parse(File.read('spec/fixtures/indexing/basic_geoblacklight.json')), 'spec/fixtures/indexing/basic_geoblacklight.json'],
-        [JSON.parse(File.read('spec/fixtures/indexing/geoblacklight.json')), 'spec/fixtures/indexing/geoblacklight.json']
+        [JSON.parse(File.read('spec/fixtures/indexing/aardvark.json')), 'spec/fixtures/indexing/aardvark.json']
       )
     end
 
-    it 'skips records with a different schema version' do
-      harvester = described_class.new(ogm_path: 'spec/fixtures/indexing/', schema_version: 'Aardvark', logger:)
+    it 'can yield JSON records for a different schema version' do
+      harvester = described_class.new(ogm_path: 'spec/fixtures/indexing/', schema_version: '1.0', logger:)
       expect { |b| harvester.docs_to_index(&b) }.to yield_successive_args(
-        [JSON.parse(File.read('spec/fixtures/indexing/aardvark.json')), 'spec/fixtures/indexing/aardvark.json']
+        [JSON.parse(File.read('spec/fixtures/indexing/basic_geoblacklight.json')), 'spec/fixtures/indexing/basic_geoblacklight.json'],
+        [JSON.parse(File.read('spec/fixtures/indexing/geoblacklight.json')), 'spec/fixtures/indexing/geoblacklight.json']
       )
     end
+
+    # Collect the titles of every record yielded, regardless of schema field
+    def yielded_titles(harvester)
+      harvester.docs_to_index.map { |record, _path| record['dct_title_s'] || record['dc_title_s'] }
+    end
+
+    context 'when skip_restricted is true (the default)' do
+      it 'skips records with restricted access rights' do
+        expect(yielded_titles(harvester)).not_to include('Property Lot, Arlington County, VA ')
+      end
+
+      it 'skips gbl1 records with restricted access rights' do
+        harvester = described_class.new(ogm_path: 'spec/fixtures/indexing', schema_version: '1.0', logger:)
+        expect(yielded_titles(harvester)).not_to include('Global Navigation and Planning Chart, Sheet 21 W ')
+      end
+    end
+
+    context 'when skip_restricted is false' do
+      it 'yields records with restricted access rights' do
+        harvester = described_class.new(ogm_path: 'spec/fixtures/indexing', skip_restricted: false, logger:)
+        expect(yielded_titles(harvester)).to include('Property Lot, Arlington County, VA ')
+      end
+
+      it 'yields gbl1 records with restricted access rights' do
+        harvester = described_class.new(ogm_path: 'spec/fixtures/indexing', schema_version: '1.0', skip_restricted: false, logger:)
+        expect(yielded_titles(harvester)).to include('Global Navigation and Planning Chart, Sheet 21 W ')
+      end
+
+      it 'can read from the OGM_SKIP_RESTRICTED environment variable' do
+        stub_const('ENV', ENV.to_h.merge('OGM_SKIP_RESTRICTED' => 'false'))
+        harvester = described_class.new(ogm_path: 'spec/fixtures/indexing', logger:)
+        expect(yielded_titles(harvester)).to include('Property Lot, Arlington County, VA ')
+      end
+    end
   end
 
   describe '#pull' do
@@ -79,15 +114,27 @@ RSpec.describe GeoCombine::Harvester do
       expect(harvester.pull_all).to eq(%w[my-institution another-institution])
     end
 
-    it 'skips repositories in the denylist' do
+    it 'skips repositories with no schema declared' do
       harvester.pull_all
-      expect(Git).not_to have_received(:open).with('https://github.com/OpenGeoMetadata/aardvark.git')
+      expect(Git).not_to have_received(:open).with('https://github.com/OpenGeoMetadata/tool.git')
     end
 
     it 'skips archived repositories' do
       harvester.pull_all
       expect(Git).not_to have_received(:open).with('https://github.com/OpenGeoMetadata/outdated-institution.git')
     end
+
+    it 'skips repositories with no data' do
+      harvester.pull_all
+      expect(Git).not_to have_received(:open).with('https://github.com/OpenGeoMetadata/empty.git')
+    end
+
+    it 'skips repositories that are on the skip list' do
+      stub_const('ENV', ENV.to_h.merge('OGM_SKIP_REPOS' => 'my-institution,another-institution'))
+      harvester.pull_all
+      expect(Git).not_to have_received(:open).with('https://github.com/OpenGeoMetadata/my-institution.git')
+      expect(Git).not_to have_received(:open).with('https://github.com/OpenGeoMetadata/another-institution.git')
+    end
   end
 
   describe '#clone' do
@@ -115,14 +162,31 @@ RSpec.describe GeoCombine::Harvester do
       expect(Git).to have_received(:clone).exactly(2).times
     end
 
-    it 'skips repositories in the denylist' do
+    it 'skips repositories with no schema declared' do
+      harvester.clone_all
+      expect(Git).not_to have_received(:clone).with('https://github.com/OpenGeoMetadata/tool.git')
+    end
+
+    it 'skips archived repositories' do
       harvester.clone_all
-      expect(Git).not_to have_received(:clone).with('https://github.com/OpenGeoMetadata/aardvark.git')
+      expect(Git).not_to have_received(:clone).with('https://github.com/OpenGeoMetadata/outdated-institution.git')
+    end
+
+    it 'skips repositories with no data' do
+      harvester.clone_all
+      expect(Git).not_to have_received(:clone).with('https://github.com/OpenGeoMetadata/empty.git')
     end
 
     it 'returns the names of repositories cloned' do
       expect(harvester.clone_all).to eq(%w[my-institution another-institution])
     end
+
+    it 'skips repositories that are on the skip list' do
+      stub_const('ENV', ENV.to_h.merge('OGM_SKIP_REPOS' => 'my-institution,another-institution'))
+      harvester.pull_all
+      expect(Git).not_to have_received(:open).with('https://github.com/OpenGeoMetadata/my-institution.git')
+      expect(Git).not_to have_received(:open).with('https://github.com/OpenGeoMetadata/another-institution.git')
+    end
   end
 
   describe '#ogm_api_uri' do

data/spec/lib/geo_combine/iso19139_spec.rb

@@ -4,6 +4,7 @@ require 'spec_helper'
 
 RSpec.describe GeoCombine::Iso19139 do
   include XmlDocs
+
   let(:iso_object) { described_class.new(stanford_iso) }
 
   describe '#initialize' do

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: geo_combine
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.11.0
 platform: ruby
 authors:
 - Jack Reed
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-13 00:00:00.000000000 Z
+date: 2026-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -196,30 +195,30 @@ dependencies:
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.25'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.25'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.8'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rubocop-rake
   requirement: !ruby/object:Gem::Requirement
@@ -269,6 +268,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
 - ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
@@ -279,6 +279,7 @@ files:
 - README.md
 - Rakefile
 - geo_combine.gemspec
+- lib/.DS_Store
 - lib/geo_combine.rb
 - lib/geo_combine/bounding_box.rb
 - lib/geo_combine/ckan_metadata.rb
@@ -304,6 +305,7 @@ files:
 - lib/xslt/fgdc2html.xsl
 - lib/xslt/iso2geoBL.xsl
 - lib/xslt/iso2html.xsl
+- lib/xslt/isoAardvark.xsl
 - lib/xslt/utils/convert-enumerations.xsl
 - lib/xslt/utils/convert-latlong.xsl
 - lib/xslt/utils/decode-uri/base.css
@@ -338,6 +340,8 @@ files:
 - spec/fixtures/indexing/basic_geoblacklight.json
 - spec/fixtures/indexing/geoblacklight.json
 - spec/fixtures/indexing/layers.json
+- spec/fixtures/indexing/restricted_aardvark.json
+- spec/fixtures/indexing/restricted_geoblacklight.json
 - spec/fixtures/indexing/test.txt
 - spec/fixtures/json_docs.rb
 - spec/fixtures/xml_docs.rb
@@ -361,7 +365,6 @@ homepage: ''
 licenses:
 - Apache
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -376,8 +379,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
-signing_key:
+rubygems_version: 3.6.2
 specification_version: 4
 summary: A Ruby toolkit for managing geospatial metadata
 test_files:
@@ -402,6 +404,8 @@ test_files:
 - spec/fixtures/indexing/basic_geoblacklight.json
 - spec/fixtures/indexing/geoblacklight.json
 - spec/fixtures/indexing/layers.json
+- spec/fixtures/indexing/restricted_aardvark.json
+- spec/fixtures/indexing/restricted_geoblacklight.json
 - spec/fixtures/indexing/test.txt
 - spec/fixtures/json_docs.rb
 - spec/fixtures/xml_docs.rb