geo_combine 0.1.0 → 0.5.0

data/spec/lib/geo_combine/formatting_spec.rb

@@ -19,4 +19,10 @@ RSpec.describe GeoCombine::Formatting do
       expect(subject.sanitize_and_remove_lines(dirty)).to_not match(/\n/)
     end
   end
+  describe '#sluggify' do
+    let(:preslug) { 'HARVARD...Co_0l' }
+    it 'handles multiple . and _ and uppercase' do
+      expect(subject.sluggify(preslug)).to eq 'harvard-co-0l'
+    end
+  end
 end

data/spec/lib/geo_combine/geo_blacklight_harvester_spec.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'rsolr'
+
+RSpec.describe GeoCombine::GeoBlacklightHarvester do
+  let(:site_key) { :INSTITUTION }
+  let(:stub_json_response) { '{}' }
+  let(:stub_solr_connection) { double('RSolr::Connection') }
+  subject(:harvester) { described_class.new(site_key) }
+
+  before do
+    allow(described_class).to receive(:config).and_return({
+      INSTITUTION: {
+        host: 'https://example.com/',
+        params: {
+          f: { dct_provenance_s: ['INSTITUTION'] }
+        }
+      }
+    })
+  end
+
+  describe 'initialization' do
+    context 'when an unconfigured site is sent in' do
+      let(:site_key) { 'unknown' }
+
+      it { expect { harvester }.to raise_error(ArgumentError) }
+    end
+  end
+
+  describe '#index' do
+    before do
+      expect(Net::HTTP).to receive(:get).with(
+        URI('https://example.com?f%5Bdct_provenance_s%5D%5B%5D=INSTITUTION&format=json&per_page=100&page=1')
+      ).and_return(stub_json_response)
+      allow(RSolr).to receive(:connect).and_return(stub_solr_connection)
+    end
+
+    let(:docs) { [{ layer_slug_s: 'abc-123' }, { layer_slug_s: 'abc-321' }] }
+    let(:stub_json_response) do
+      { response: { docs: docs, pages: { current_page: 1, total_pages: 1 } } }.to_json
+    end
+
+    it 'adds documents returned to solr' do
+      expect(stub_solr_connection).to receive(:update).with(hash_including(data: docs.to_json)).and_return(nil)
+      harvester.index
+    end
+
+    describe 'document tranformations' do
+      let(:docs) do
+        [
+         { layer_slug_s: 'abc-123', _version_: '1', timestamp: '1999-12-31', score: 0.1 },
+         { layer_slug_s: 'abc-321', dc_source_s: 'abc-123' }
+        ]
+      end
+
+      context 'when a tranformer is set' do
+        before do
+          expect(described_class).to receive(:document_transformer).at_least(:once).and_return(
+            ->(doc) {
+              doc.delete('_version_')
+              doc
+            }
+          )
+        end
+
+        it 'removes the _version_ field as requested' do
+          expect(stub_solr_connection).to receive(:update).with(
+            hash_including(
+              data: [
+                { layer_slug_s: 'abc-123', timestamp: '1999-12-31', score: 0.1 },
+                { layer_slug_s: 'abc-321', dc_source_s: 'abc-123' }
+              ].to_json
+            )
+          ).and_return(nil)
+
+          harvester.index
+        end
+      end
+
+      context 'when no transformer is set' do
+        it 'removes the _version_, timestamp, and score fields' do
+          expect(stub_solr_connection).to receive(:update).with(
+            hash_including(
+              data: [
+                { layer_slug_s: 'abc-123' },
+                { layer_slug_s: 'abc-321', dc_source_s: 'abc-123' }
+              ].to_json
+            )
+          ).and_return(nil)
+
+          harvester.index
+        end
+      end
+    end
+  end
+
+  describe 'BlacklightResponseVersionFactory' do
+    let(:version_class) { described_class::BlacklightResponseVersionFactory.call(json) }
+
+    context 'when a legacy blacklight version (6 and earlier)' do
+      let(:json) { { 'response' => {} } }
+
+      it { expect(version_class).to be described_class::LegacyBlacklightResponse }
+    end
+
+    context 'when a modern blacklight version (7 and later)' do
+      let(:json) { { 'links' => {}, 'data' => [] } }
+
+      it { expect(version_class).to be described_class::ModernBlacklightResponse }
+    end
+
+    context 'when a the JSON response is not recognizable' do
+      let(:json) { { error: 'Broken' } }
+
+      it { expect { version_class }.to raise_error(NotImplementedError) }
+    end
+  end
+
+  describe 'LegacyBlacklightResponse' do
+    before do
+      allow(RSolr).to receive(:connect).and_return(stub_solr_connection)
+    end
+
+    let(:first_docs) {  [{ 'layer_slug_s' => 'abc-123' }, { 'layer_slug_s' => 'abc-321' }] }
+    let(:second_docs) { [{ 'layer_slug_s' => 'xyz-123' }, { 'layer_slug_s' => 'xyz-321' }] }
+    let(:stub_first_response) do
+      { 'response' => { 'docs' => first_docs, 'pages' =>  { 'current_page' => 1, 'total_pages' => 2 } } }
+    end
+    let(:stub_second_response) do
+      { 'response' => { 'docs' => second_docs, 'pages' => { 'current_page' => 2, 'total_pages' => 2 } } }
+    end
+
+    describe '#documents' do
+      it 'pages through the response and returns all the documents' do
+        expect(Net::HTTP).to receive(:get).with(
+          URI('https://example.com?f%5Bdct_provenance_s%5D%5B%5D=INSTITUTION&format=json&per_page=100&page=2')
+        ).and_return(stub_second_response.to_json)
+        base_url = 'https://example.com?f%5Bdct_provenance_s%5D%5B%5D=INSTITUTION&format=json&per_page=100'
+        docs = described_class::LegacyBlacklightResponse.new(response: stub_first_response, base_url: base_url).documents
+
+        expect(docs.to_a).to eq([first_docs, second_docs])
+      end
+    end
+  end
+
+  describe 'ModernBlacklightResponse' do
+    before do
+      allow(RSolr).to receive(:connect).and_return(stub_solr_connection)
+      expect(Net::HTTP).to receive(:get).with(
+        URI('https://example.com?f%5Bdct_provenance_s%5D%5B%5D=INSTITUTION&format=json&per_page=100&page=2')
+      ).and_return(second_results_response.to_json)
+    end
+
+    let(:first_results_response) do
+      { 'data' => [
+          { 'links' => { 'self' => 'https://example.com/catalog/abc-123' } },
+          { 'links' => { 'self' => 'https://example.com/catalog/abc-321' } }
+        ],
+        'links' => { 'next' => 'https://example.com?f%5Bdct_provenance_s%5D%5B%5D=INSTITUTION&format=json&per_page=100&page=2' }
+      }
+    end
+
+    let(:second_results_response) do
+      { 'data' => [
+          { 'links' => { 'self' => 'https://example.com/catalog/xyz-123' } },
+          { 'links' => { 'self' => 'https://example.com/catalog/xyz-321' } }
+        ]
+      }
+    end
+
+    describe '#documents' do
+      it 'pages through the response and fetches documents for each "link" on the response data' do
+        %w[abc-123 abc-321 xyz-123 xyz-321].each do |id|
+          expect(Net::HTTP).to receive(:get).with(
+            URI("https://example.com/catalog/#{id}/raw")
+          ).and_return({ 'layer_slug_s' => id }.to_json)
+        end
+
+        base_url = 'https://example.com?f%5Bdct_provenance_s%5D%5B%5D=INSTITUTION&format=json&per_page=100'
+        docs = described_class::ModernBlacklightResponse.new(response: first_results_response, base_url: base_url).documents
+
+        expect(docs.to_a).to eq([
+          [{ 'layer_slug_s' => 'abc-123' }, { 'layer_slug_s' => 'abc-321' }],
+          [{ 'layer_slug_s' => 'xyz-123' }, { 'layer_slug_s' => 'xyz-321' }],
+        ])
+      end
+    end
+  end
+end

data/spec/lib/geo_combine/geoblacklight_spec.rb

@@ -3,16 +3,18 @@ require 'spec_helper'
 RSpec.describe GeoCombine::Geoblacklight do
   include XmlDocs
   include JsonDocs
-  let(:full_geobl) { GeoCombine::Geoblacklight.new(full_geoblacklight) }
-  let(:basic_geobl) { GeoCombine::Geoblacklight.new(basic_geoblacklight) }
+  include GeoCombine::Exceptions
+  let(:full_geobl) { described_class.new(full_geoblacklight) }
+  let(:basic_geobl) { described_class.new(basic_geoblacklight) }
+  let(:pre_v1_geobl) { described_class.new(geoblacklight_pre_v1) }
   describe '#initialize' do
     it 'parses metadata argument JSON to Hash' do
       expect(basic_geobl.instance_variable_get(:@metadata)).to be_an Hash
     end
     describe 'merges fields argument into metadata' do
-      let(:basic_geobl) { GeoCombine::Geoblacklight.new(basic_geoblacklight, 'uuid' => 'new one', "extra_field" => true)}
+      let(:basic_geobl) { described_class.new(basic_geoblacklight, 'dc_identifier_s' => 'new one', "extra_field" => true)}
       it 'overwrites existing metadata fields' do
-        expect(basic_geobl.metadata['uuid']).to eq 'new one'
+        expect(basic_geobl.metadata['dc_identifier_s']).to eq 'new one'
       end
       it 'adds in new fields' do
         expect(basic_geobl.metadata['extra_field']).to be true
@@ -22,7 +24,7 @@ RSpec.describe GeoCombine::Geoblacklight do
   describe '#metadata' do
     it 'reads the metadata instance variable' do
       expect(basic_geobl.metadata).to be_an Hash
-      expect(basic_geobl.metadata).to have_key 'uuid'
+      expect(basic_geobl.metadata).to have_key 'dc_identifier_s'
     end
   end
   describe '#to_json' do
@@ -31,13 +33,9 @@ RSpec.describe GeoCombine::Geoblacklight do
       expect(valid_json?(basic_geobl.to_json)).to be_truthy
     end
   end
+  let(:enhanced_geobl) { described_class.new(basic_geoblacklight, 'layer_geom_type_s' => 'esriGeometryPolygon') }
+  before { enhanced_geobl.enhance_metadata }
   describe '#enhance_metadata' do
-    let(:enhanced_geobl) { GeoCombine::Geoblacklight.new(basic_geoblacklight, 'dct_references_s' => '', 'layer_geom_type_s' => 'esriGeometryPolygon') }
-    before { enhanced_geobl.enhance_metadata }
-    it 'calls enhancement methods to validate document' do
-      expect { basic_geobl.valid? }.to raise_error JSON::Schema::ValidationError
-      expect(enhanced_geobl.valid?).to be true
-    end
     it 'enhances the dc_subject_sm field' do
       expect(enhanced_geobl.metadata['dc_subject_sm']).to include 'Boundaries', 'Inland Waters'
     end
@@ -52,8 +50,136 @@ RSpec.describe GeoCombine::Geoblacklight do
     it 'a valid geoblacklight-schema document should be valid' do
       expect(full_geobl.valid?).to be true
     end
+    context 'must have required fields' do
+      %w(
+        dc_title_s
+        dc_identifier_s
+        dc_rights_s
+        dct_provenance_s
+        layer_slug_s
+        solr_geom
+      ).each do |field|
+        it field do
+          full_geobl.metadata.delete field
+          expect { full_geobl.valid? }.to raise_error(JSON::Schema::ValidationError, /#{field}/)
+        end
+      end
+    end
+    context 'need not have optional fields' do
+      %w(
+        dc_description_s
+        dc_format_s
+        dc_language_s
+        dc_publisher_s
+        dc_source_sm
+        dc_subject_sm
+        dct_isPartOf_sm
+        dct_issued_dt
+        dct_references_s
+        dct_spatial_sm
+        dct_temporal_sm
+        geoblacklight_version
+        layer_geom_type_s
+        layer_id_s
+        layer_modified_dt
+        solr_year_i
+      ).each do |field|
+        it field do
+          full_geobl.metadata.delete field
+          expect { full_geobl.valid? }.not_to raise_error
+        end
+      end
+    end
+    context 'need not have deprecated fields' do
+      %w(
+        dc_relation_sm
+        dc_type_s
+        georss_box_s
+        georss_point_s
+        uuid
+      ).each do |field|
+        it field do
+          full_geobl.metadata.delete field
+          expect { full_geobl.valid? }.not_to raise_error
+        end
+      end
+    end
     it 'an invalid document' do
       expect { basic_geobl.valid? }.to raise_error JSON::Schema::ValidationError
     end
+    it 'calls the dct_references_s validator' do
+      expect(enhanced_geobl).to receive(:dct_references_validate!)
+      enhanced_geobl.valid?
+    end
+    it 'validates spatial bounding box' do
+      expect(JSON::Validator).to receive(:validate!).and_return true
+      expect { basic_geobl.valid? }
+        .to raise_error GeoCombine::Exceptions::InvalidGeometry
+    end
+  end
+  describe '#dct_references_validate!' do
+    context 'with valid document' do
+      it 'is valid' do
+        expect(full_geobl.dct_references_validate!).to be true
+      end
+    end
+    context 'with invalid document' do
+      let(:unparseable_json) do
+        <<-JSON
+          {
+            \"http://schema.org/url\":\"http://example.com/abc123\",,
+            \"http://schema.org/downloadUrl\":\"http://example.com/abc123/data.zip\"
+          }
+        JSON
+      end
+      let(:bad_ref) do
+        GeoCombine::Geoblacklight.new(
+          basic_geoblacklight, 'dct_references_s' => unparseable_json, 'layer_geom_type_s' => 'esriGeometryPolygon'
+        )
+      end
+      let(:not_hash) do
+        GeoCombine::Geoblacklight.new(
+          basic_geoblacklight, 'dct_references_s' => '[{}]', 'layer_geom_type_s' => 'esriGeometryPolygon'
+        )
+      end
+      before do
+        bad_ref.enhance_metadata
+        not_hash.enhance_metadata
+      end
+      it 'unparseable json' do
+        expect { bad_ref.dct_references_validate! }.to raise_error JSON::ParserError
+      end
+      it 'not a hash' do
+        expect { not_hash.dct_references_validate! }.to raise_error GeoCombine::Exceptions::InvalidDCTReferences
+      end
+    end
+  end
+  describe 'spatial_validate!' do
+    context 'when valid' do
+      it { full_geobl.spatial_validate! }
+    end
+    context 'when invalid' do
+      it { expect { basic_geobl.spatial_validate! }.to raise_error GeoCombine::Exceptions::InvalidGeometry }
+    end
+  end
+  describe 'upgrade_to_v1' do
+    before do
+      expect(pre_v1_geobl).to receive(:upgrade_to_v1).and_call_original
+      pre_v1_geobl.enhance_metadata
+    end
+
+    it 'tags with version' do
+      expect(pre_v1_geobl.metadata).to include('geoblacklight_version' => '1.0')
+    end
+
+    it 'properly deprecates fields' do
+      described_class::DEPRECATED_KEYS_V1.each do |k|
+        expect(pre_v1_geobl.metadata.keys).not_to include(k.to_s)
+      end
+    end
+
+    it 'normalizes slugs' do
+      expect(pre_v1_geobl.metadata).to include('layer_slug_s' => 'sde-columbia-esri-arcatlas-snow-ln')
+    end
   end
 end

data/spec/lib/geo_combine/iso19139_spec.rb

@@ -19,7 +19,7 @@ RSpec.describe GeoCombine::Iso19139 do
     end
   end
   describe '#to_geoblacklight' do
-    let(:valid_geoblacklight) { iso_object.to_geoblacklight('layer_geom_type_s' => 'Polygon', 'dct_references_s' => '') }
+    let(:valid_geoblacklight) { iso_object.to_geoblacklight('layer_geom_type_s' => 'Polygon') }
     it 'should create a GeoCombine::Geoblacklight object' do
       expect(valid_geoblacklight).to be_an GeoCombine::Geoblacklight
     end
@@ -27,6 +27,9 @@ RSpec.describe GeoCombine::Iso19139 do
       valid_geoblacklight.enhance_metadata
       expect(valid_geoblacklight.valid?).to be_truthy
     end
+    it 'should have geoblacklight_version' do
+      expect(valid_geoblacklight.metadata['geoblacklight_version']).to eq '1.0'
+    end
     it 'should have dc_creator_sm' do
       expect(valid_geoblacklight.metadata["dc_creator_sm"]).to be_an Array
       expect(valid_geoblacklight.metadata["dc_creator_sm"]).to eq ["Circuit Rider Productions"]
@@ -34,7 +37,7 @@ RSpec.describe GeoCombine::Iso19139 do
     it 'should have dc_publisher_sm' do
       expect(valid_geoblacklight.metadata["dc_publisher_sm"]).to be_an Array
       expect(valid_geoblacklight.metadata["dc_publisher_sm"]).to eq ["Circuit Rider Productions"]
-    end    
+    end
   end
   describe '#to_html' do
     it 'should create a transformation of the metadata as a String' do

data/spec/lib/geo_combine/ogp_spec.rb

@@ -0,0 +1,163 @@
+require 'spec_helper'
+
+RSpec.describe GeoCombine::OGP do
+  include JsonDocs
+
+  subject(:ogp) { GeoCombine::OGP.new(ogp_harvard_raster) }
+  let(:ogp_tufts) { GeoCombine::OGP.new(ogp_tufts_vector) }
+  let(:ogp_line) { GeoCombine::OGP.new(ogp_harvard_line) }
+  let(:metadata) { ogp.instance_variable_get(:@metadata) }
+
+  describe '#initialize' do
+    it 'parses JSON into metadata Hash' do
+      expect(metadata).to be_an Hash
+    end
+  end
+
+  describe '#to_geoblacklight' do
+    it 'calls geoblacklight_terms to create a GeoBlacklight object' do
+      expect(ogp).to receive(:geoblacklight_terms).and_return({})
+      expect(ogp.to_geoblacklight).to be_an GeoCombine::Geoblacklight
+    end
+  end
+
+  describe '#geoblacklight_terms' do
+    describe 'builds a hash which maps metadata' do
+      it 'with dc_identifier_s' do
+        expect(ogp.geoblacklight_terms).to include(dc_identifier_s: 'HARVARD.SDE2.G1059_W57_1654_PF_SH1')
+      end
+      it 'with dc_title_s' do
+        expect(ogp.geoblacklight_terms).to include(dc_title_s: 'World Map, 1654 (Raster Image)')
+      end
+      it 'with dc_description_s sanitized' do
+        expect(ogp.geoblacklight_terms).to include(dc_description_s: metadata['Abstract'])
+      end
+      it 'with dc_rights_s' do
+        expect(ogp.geoblacklight_terms).to include(dc_rights_s: 'Public')
+        expect(ogp_line.geoblacklight_terms).to include(dc_rights_s: 'Restricted')
+      end
+      it 'with dct_provenance_s' do
+        expect(ogp.geoblacklight_terms).to include(dct_provenance_s: 'Harvard')
+      end
+      it 'with dct_references_s' do
+        expect(ogp.geoblacklight_terms).to include(:dct_references_s)
+      end
+      it 'with layer_id_s that is blank' do
+        expect(ogp.geoblacklight_terms)
+          .to include(layer_id_s: "#{metadata['WorkspaceName']}:#{metadata['Name']}")
+      end
+      it 'with layer_geom_type_s' do
+        expect(ogp.geoblacklight_terms).to include(:layer_geom_type_s)
+      end
+      it 'with layer_slug_s' do
+        expect(ogp.geoblacklight_terms)
+          .to include(layer_slug_s: 'harvard-g1059-w57-1654-pf-sh1')
+      end
+      it 'with solr_geom' do
+        expect(ogp.geoblacklight_terms).to include(:solr_geom)
+      end
+      it 'with dc_subject_sm' do
+        expect(ogp.geoblacklight_terms).to include(
+          dc_subject_sm: [
+            'Maps', 'Human settlements', 'Cities and towns', 'Villages',
+            'Bodies of water', 'Landforms', 'Transportation',
+            'imageryBaseMapsEarthCover'
+          ]
+        )
+      end
+      it 'with dct_spatial_sm' do
+        expect(ogp.geoblacklight_terms).to include(
+          dct_spatial_sm: [
+            'Earth', 'Northern Hemisphere', 'Southern Hemisphere',
+            'Eastern Hemisphere', 'Western Hemisphere', 'Africa', 'Asia',
+            'Australia', 'Europe', 'North America', 'South America',
+            'Arctic regions'
+          ]
+        )
+      end
+    end
+  end
+
+  describe '#ogp_geom' do
+    it 'when Paper Map use Raster' do
+      expect(ogp.ogp_geom).to eq 'Raster'
+    end
+    it 'anything else, return it' do
+      expect(ogp_tufts.ogp_geom).to eq 'Polygon'
+    end
+  end
+
+  describe '#ogp_formats' do
+    context 'when Paper Map or Raster' do
+      it 'returns GeoTIFF' do
+        %w[Raster Paper\ Map].each do |datatype|
+          expect(ogp).to receive(:metadata).and_return('DataType' => datatype)
+          expect(ogp.ogp_formats).to eq 'GeoTIFF'
+        end
+
+      end
+    end
+    context 'when Polygon, Line, or Point' do
+      it 'returns Shapefile' do
+        %w[Polygon Line Point].each do |datatype|
+          expect(ogp).to receive(:metadata).and_return('DataType' => datatype)
+          expect(ogp.ogp_formats).to eq 'Shapefile'
+        end
+      end
+    end
+    context 'unknown data types' do
+      it 'throws exception' do
+        expect(ogp).to receive(:metadata).twice.and_return('DataType' => 'Unknown')
+        expect { ogp.ogp_formats }.to raise_error(ArgumentError, 'Unknown')
+      end
+    end
+  end
+
+  describe '#envelope' do
+    it 'properly formatted envelope' do
+      expect(ogp.envelope).to eq 'ENVELOPE(-180.0, 180.0, 90.0, -90.0)'
+    end
+    it 'fails on out-of-bounds envelopes' do
+      expect(ogp).to receive(:west).and_return(-181)
+      expect { ogp.envelope }.to raise_error(ArgumentError)
+    end
+  end
+
+  describe '#references' do
+    context 'harvard raster' do
+      it 'has wms and download services' do
+        expect(JSON.parse(ogp.references)).to include(
+          'http://www.opengis.net/def/serviceType/ogc/wms' => 'http://pelham.lib.harvard.edu:8080/geoserver/wms',
+          'http://schema.org/DownloadAction' => 'http://pelham.lib.harvard.edu:8080/HGL/HGLOpenDelivery'
+        )
+      end
+    end
+    context 'tufts vector' do
+      it 'has wms wfs services' do
+        expect(JSON.parse(ogp_tufts.references)).to include(
+          'http://www.opengis.net/def/serviceType/ogc/wms' => 'http://geoserver01.uit.tufts.edu/wms',
+          'http://www.opengis.net/def/serviceType/ogc/wfs' => 'http://geoserver01.uit.tufts.edu/wfs'
+        )
+      end
+    end
+    context 'harvard line' do
+      it 'has restricted services' do
+        expect(JSON.parse(ogp_line.references)).to include(
+          'http://www.opengis.net/def/serviceType/ogc/wfs' => 'http://hgl.harvard.edu:8080/geoserver/wfs',
+          'http://www.opengis.net/def/serviceType/ogc/wms' => 'http://hgl.harvard.edu:8080/geoserver/wms'
+        )
+        expect(JSON.parse(ogp_line.references)).not_to include('http://schema.org/DownloadAction')
+      end
+    end
+  end
+
+  describe 'valid geoblacklight schema' do
+    context 'harvard' do
+      it { expect { ogp.to_geoblacklight.valid? }.to_not raise_error }
+      it { expect { ogp_line.to_geoblacklight.valid? }.to_not raise_error }
+    end
+    context 'tufts' do
+      it { expect { ogp_tufts.to_geoblacklight.valid? }.to_not raise_error }
+    end
+  end
+end