geo_combine 0.4.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 0370f82160756d74fe36f4c4a5e9639b8e06d461
-  data.tar.gz: e06d158e0770862c7033935c91c82bfe6b9d9f21
+SHA256:
+  metadata.gz: ba7bc8217102aecc80fc75e0ca8915292b0be85c18bac51a2f19b486d33273a0
+  data.tar.gz: f66c6e54dedf27a4e8786e2bbe1aa449d1aa5c27e1f5fe8978b7dd75206fc496
 SHA512:
-  metadata.gz: f5072b677855334cbe0716a3639394ead6c10bce24cdec3485daf5764d36df1466938247075cbf5737070a7b08fb5f1926aa618ac0c9679783c9edf8e5ea7b86
-  data.tar.gz: 26098a941f59b4c36fa0c4f8331d2c31d8575c2978e4bcba1804a9711b98a22e34d78e16dfa1a5d04da4a130f887bb91ff5bc4f0de794bb2dc2e07ffa72b93d2
+  metadata.gz: 5d191f0f24795e8544a9b231116fedb4ba635ed3dba333de66f60ccaca5776fdaddb3d399be25a77112cac370d1f2eab36ab1819a7216eb5175678eb4b9aae29
+  data.tar.gz: 0d45aec320e4c543c349608d36cfb632197fedb8571adabd77ab8e11d216c1922a5dd135862b3de51cbce70a3ab32089c67561f4bd9c16458eeb864347e46011

data/.github/workflows/ruby.yml

@@ -0,0 +1,53 @@
+name: CI
+
+on: [push]
+
+jobs:
+  rubocop:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.7
+    - name: Install dependencies
+      run: bundle install
+    - name: Run linter
+      run: bundle exec rubocop
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: [2.7, 3.0, 3.1]
+        faraday_version: [''] # Defaults to whatever's the most recent version.
+        include:
+          - ruby: 2.7
+            faraday_version: '~> 1.0'
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+
+    - name: Install bundler
+      run: gem install bundler -v 2.1.1
+
+    - name: Install dependencies
+      run: bundle _2.1.1_ install
+      env:
+        FARADAY_VERSION: ${{ matrix.faraday_version }}
+
+    - name: Run tests
+      run: bundle exec rake spec
+      env:
+        FARADAY_VERSION: ${{ matrix.faraday_version }}
+
+    - name: Upload coverage artifacts
+      uses: actions/upload-artifact@v2
+      with:
+        name: coverage
+        path: coverage/

data/.gitignore

@@ -12,3 +12,5 @@
 *.o
 *.a
 mkmf.log
+.tool-versions
+.byebug_history

data/.rubocop.yml

@@ -0,0 +1,20 @@
+require:
+  - rubocop-rspec
+  - rubocop-rake
+
+inherit_from: .rubocop_todo.yml
+
+AllCops:
+  TargetRubyVersion: 2.7
+  DisplayCopNames: true
+  NewCops: enable
+  Exclude:
+  - 'geo_combine.gemspec'
+  - 'tmp/**/*'
+
+RSpec/DescribeClass:
+  Enabled: false
+
+RSpec/BeforeAfterAll:
+  Exclude:
+    - 'spec/lib/tasks/geo_combine_spec.rb'

data/.rubocop_todo.yml

@@ -0,0 +1,165 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2022-02-17 18:38:52 UTC using RuboCop version 1.25.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+Lint/RescueException:
+  Exclude:
+    - 'spec/helpers.rb'
+
+# Offense count: 1
+Lint/UselessAssignment:
+  Exclude:
+    - 'spec/helpers.rb'
+
+# Offense count: 7
+# Configuration parameters: IgnoredMethods, CountRepeatedAttributes.
+Metrics/AbcSize:
+  Max: 33
+
+# Offense count: 25
+# Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
+# IgnoredMethods: refine
+Metrics/BlockLength:
+  Max: 181
+
+# Offense count: 1
+# Configuration parameters: CountComments, CountAsOne.
+Metrics/ClassLength:
+  Max: 152
+
+# Offense count: 3
+# Configuration parameters: IgnoredMethods.
+Metrics/CyclomaticComplexity:
+  Max: 11
+
+# Offense count: 10
+# Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
+Metrics/MethodLength:
+  Max: 21
+
+# Offense count: 1
+# Configuration parameters: CountComments, CountAsOne.
+Metrics/ModuleLength:
+  Max: 1657
+
+# Offense count: 1
+# Configuration parameters: IgnoredMethods.
+Metrics/PerceivedComplexity:
+  Max: 11
+
+# Offense count: 9
+# Configuration parameters: Prefixes.
+# Prefixes: when, with, without
+RSpec/ContextWording:
+  Exclude:
+    - 'spec/lib/geo_combine/geoblacklight_spec.rb'
+    - 'spec/lib/geo_combine/ogp_spec.rb'
+
+# Offense count: 9
+# Configuration parameters: CountAsOne.
+RSpec/ExampleLength:
+  Max: 12
+
+# Offense count: 4
+RSpec/ExpectInHook:
+  Exclude:
+    - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
+    - 'spec/lib/geo_combine/geoblacklight_spec.rb'
+
+# Offense count: 1
+# Configuration parameters: Include, CustomTransform, IgnoreMethods, SpecSuffixOnly.
+# Include: **/*_spec*rb*, **/spec/**/*
+RSpec/FilePath:
+  Exclude:
+    - 'spec/lib/geo_combine_spec.rb'
+
+# Configuration parameters: .
+# SupportedStyles: have_received, receive
+RSpec/MessageSpies:
+  EnforcedStyle: have_received
+  Exclude:
+    - 'spec/lib/geo_combine/esri_open_data_spec.rb'
+    - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
+    - 'spec/lib/geo_combine/geoblacklight_spec.rb'
+    - 'spec/lib/geo_combine/ogp_spec.rb'
+    - 'spec/lib/geo_combine_spec.rb'
+
+# Offense count: 39
+RSpec/MultipleExpectations:
+  Max: 5
+
+# Offense count: 3
+# Configuration parameters: AllowSubject.
+RSpec/MultipleMemoizedHelpers:
+  Max: 7
+
+# Offense count: 5
+# Configuration parameters: IgnoreSharedExamples.
+RSpec/NamedSubject:
+  Exclude:
+    - 'spec/lib/geo_combine/formatting_spec.rb'
+
+# Offense count: 8
+RSpec/NestedGroups:
+  Max: 4
+
+# Offense count: 1
+RSpec/OverwritingSetup:
+  Exclude:
+    - 'spec/lib/geo_combine/geoblacklight_spec.rb'
+
+# Offense count: 2
+RSpec/RepeatedExampleGroupBody:
+  Exclude:
+    - 'spec/lib/geo_combine/iso19139_spec.rb'
+
+# Offense count: 19
+RSpec/StubbedMock:
+  Exclude:
+    - 'spec/lib/geo_combine/esri_open_data_spec.rb'
+    - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
+    - 'spec/lib/geo_combine/geoblacklight_spec.rb'
+    - 'spec/lib/geo_combine/ogp_spec.rb'
+    - 'spec/lib/geo_combine_spec.rb'
+
+# Offense count: 5
+RSpec/SubjectStub:
+  Exclude:
+    - 'spec/lib/geo_combine/ogp_spec.rb'
+
+# Offense count: 1
+# Configuration parameters: IgnoreNameless, IgnoreSymbolicNames.
+RSpec/VerifiedDoubles:
+  Exclude:
+    - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
+
+# Offense count: 1
+Security/Open:
+  Exclude:
+    - 'lib/geo_combine/geoblacklight.rb'
+
+# Offense count: 7
+# Configuration parameters: AllowedConstants.
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'lib/geo_combine/bounding_box.rb'
+    - 'lib/geo_combine/ckan_metadata.rb'
+    - 'lib/geo_combine/cli.rb'
+    - 'lib/geo_combine/geo_blacklight_harvester.rb'
+    - 'lib/geo_combine/geoblacklight.rb'
+    - 'lib/geo_combine/geometry_types.rb'
+    - 'lib/geo_combine/iso19139.rb'
+
+# Offense count: 7
+# Cop supports --auto-correct.
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Layout/LineLength:
+  Max: 159

data/Gemfile

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in geo_combine.gemspec
 gemspec
 
-gem 'coveralls', require: false
 gem 'byebug'
+gem 'faraday', ENV['FARADAY_VERSION'] if ENV['FARADAY_VERSION'].to_s != ''

data/README.md

@@ -1,6 +1,8 @@
 # GeoCombine
 
-[![Build Status](https://travis-ci.org/OpenGeoMetadata/GeoCombine.svg?branch=master)](https://travis-ci.org/OpenGeoMetadata/GeoCombine) | [![Coverage Status](https://coveralls.io/repos/OpenGeoMetadata/GeoCombine/badge.svg?branch=master)](https://coveralls.io/r/OpenGeoMetadata/GeoCombine?branch=master)
+   ![CI](https://github.com/OpenGeoMetadata/GeoCombine/actions/workflows/ruby.yml/badge.svg)
+| [![Coverage Status](https://img.shields.io/badge/coverage-95%25-brightgreen)]()
+| [![Gem Version](https://img.shields.io/gem/v/geo_combine.svg)](https://github.com/OpenGeoMetadata/GeoCombine/releases)
 
 
 A Ruby toolkit for managing geospatial metadata, including:
@@ -61,6 +63,12 @@ You can also specify a single repository:
 $ bundle exec rake geocombine:clone[edu.stanford.purl]
 ```
 
+*Note: If you are using zsh, you will need to use escape characters in front of the brackets:*
+
+```sh
+$ bundle exec rake geocombine:clone\[edu.stanford.purl\]
+```
+
 #### Update local OpenGeoMetadata repositories
 
 ```sh
@@ -75,6 +83,12 @@ You can also specify a single repository:
 $ bundle exec rake geocombine:pull[edu.stanford.purl]
 ```
 
+*Note: If you are using zsh, you will need to use escape characters in front of the brackets:*
+
+```sh
+$ bundle exec rake geocombine:pull\[edu.stanford.purl\]
+```
+
 #### Index GeoBlacklight documents
 
 To index into Solr, GeoCombine requires a Solr instance that is running the
@@ -101,6 +115,71 @@ change the [`commitWithin` parameter](https://lucene.apache.org/solr/guide/6_6/u
 $ SOLR_COMMIT_WITHIN=100 bundle exec rake geocombine:index
 ```
 
+### Harvesting and indexing documents from GeoBlacklight sites
+
+GeoCombine provides a Harvester class and rake task to harvest and index content from GeoBlacklight sites (or any site that follows the Blacklight API format). Given that the configurations can change from consumer to consumer and site to site, the class provides a relatively simple configuration API. This can be configured in an initializer, a wrapping rake task, or any other ruby context where the rake task our class would be invoked.
+
+```sh
+bundle exec rake geocombine:geoblacklight_harvester:index[YOUR_CONFIGURED_SITE_KEY]
+```
+
+#### Harvester configuration
+
+Only the sites themselves are required to be configured but there are various configuration options that can (optionally) be supplied to modify the harvester's behavior.
+
+```ruby
+GeoCombine::GeoBlacklightHarvester.configure do
+  {
+    commit_within: '10000',
+    crawl_delay: 1, # All sites
+    debug: true,
+    SITE1: {
+      crawl_delay: 2, # SITE1 only
+      host: 'https://geoblacklight.example.edu',
+      params: {
+        f: {
+          dct_provenance_s: ['Institution']
+        }
+      }
+    },
+    SITE2: {
+      host: 'https://geoportal.example.edu',
+      params: {
+        q: '*'
+      }
+    }
+  }
+end
+```
+
+##### Crawl Delays (default: none)
+
+Crawl delays can be configured (in seconds) either globally for all sites or on a per-site basis. This will cause a delay for that number of seconds between each search results page (note that Blacklight 7 necessitates a lot of requests per results page and this only causes the delay per page of results)
+
+##### Solr's commitWithin (default: 5000 milliseconds)
+
+Solr's commitWithin option can be configured (in milliseconds) by passing a value under the commit_within key.
+
+##### Debugging (default: false)
+
+The harvester and indexer will only `puts` content when errors happen. It is possible to see some progress information by setting the debug configuration option.
+
+#### Transforming Documents
+
+You may need to transform documents that are harvested for various purposes (removing fields, adding fields, omitting a document all together, etc). You can configure some ruby code (a proc) that will take the document in, transform it, and return the transformed document. By default the indexer will remove the `score`, `timestamp`, and `_version_` fields from the documents harvested. If you provide your own transformer, you'll likely want to remove these fields in addition to the other transformations you provide.
+
+```ruby
+GeoCombine::GeoBlacklightIndexer.document_transformer = -> (document) do
+  # Removes "bogus_field" from the content we're harvesting
+  # in addition to some other solr fields we don't want
+  %w[_version_ score timestamp bogus_field].each do |field|
+    document.delete(field)
+  end
+
+  document
+end
+```
+
 ## Tests
 
 To run the tests, use:

data/Rakefile

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 require 'bundler/gem_tasks'
 
-Dir.glob('lib/tasks/*.rake').each { |r| load r}
+Dir.glob('lib/tasks/*.rake').each { |r| load r }
 
 desc 'Run console for development'
 task :console do
@@ -17,7 +19,7 @@ begin
 
   RSpec::Core::RakeTask.new(:spec)
 
-  task :default => :spec
+  task default: :spec
 rescue LoadError
   # no rspec available
 end

data/bin/geocombine

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'geo_combine/cli'
 

data/geo_combine.gemspec

@@ -30,4 +30,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rspec-html-matchers'
+  spec.add_development_dependency 'rubocop', '~> 1.25'
+  spec.add_development_dependency 'rubocop-rspec', '~> 2.8'
+  spec.add_development_dependency 'rubocop-rake'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'webmock', '~> 3.14'
 end

data/lib/geo_combine/bounding_box.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   class BoundingBox
     attr_reader :west, :south, :east, :north
@@ -24,11 +26,13 @@ module GeoCombine
     def valid?
       [south, north].map do |coord|
         next if (-90..90).cover?(coord)
+
         raise GeoCombine::Exceptions::InvalidGeometry,
               "#{coord} should be in range -90 90"
       end
       [east, west].map do |coord|
         next if (-180..180).cover?(coord)
+
         raise GeoCombine::Exceptions::InvalidGeometry,
               "#{coord} should be in range -180 180"
       end
@@ -45,7 +49,8 @@ module GeoCombine
 
     def self.from_envelope(envelope)
       return if envelope.nil?
-      envelope = envelope[/.*ENVELOPE\(([^\)]*)/, 1].split(',')
+
+      envelope = envelope[/.*ENVELOPE\(([^)]*)/, 1].split(',')
       new(
         west: envelope[0],
         south: envelope[3],
@@ -59,6 +64,7 @@ module GeoCombine
     # @param [String] delimiter "," or " "
     def self.from_string_delimiter(spatial, delimiter: ',')
       return if spatial.nil?
+
       spatial = spatial.split(delimiter)
       new(
         west: spatial[0],

data/lib/geo_combine/ckan_metadata.rb

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 module GeoCombine
   class CkanMetadata
-    MAX_STRING_LENGTH = 32765 # Solr limit
+    MAX_STRING_LENGTH = 32_765 # Solr limit
 
     attr_reader :metadata
+
     def initialize(metadata)
       @metadata = JSON.parse(metadata)
     end
@@ -31,7 +34,7 @@ module GeoCombine
         dc_subject_sm: subjects,
         dct_references_s: external_references.to_json.to_s,
         dc_format_s: downloadable? ? 'ZIP' : nil # TODO: we only allow direct ZIP file downloads
-      }.select { |_k, v| !v.nil? }
+      }.compact
     end
 
     def organization
@@ -54,7 +57,7 @@ module GeoCombine
       begin
         return bbox.to_envelope if bbox.valid?
       rescue GeoCombine::Exceptions::InvalidGeometry
-        return nil
+        nil
       end
     end
 
@@ -66,7 +69,7 @@ module GeoCombine
       begin
         return bbox.to_envelope if bbox.valid?
       rescue GeoCombine::Exceptions::InvalidGeometry
-        return nil
+        nil
       end
     end
 
@@ -87,11 +90,9 @@ module GeoCombine
         'http://schema.org/url' => resource_urls('information').first
       }
 
-      if downloadable?
-        h['http://schema.org/downloadUrl'] = resource_urls('download').first
-      end
+      h['http://schema.org/downloadUrl'] = resource_urls('download').first if downloadable?
 
-      h.select { |_k, v| !v.nil? }
+      h.compact
     end
 
     def downloadable?
@@ -100,6 +101,7 @@ module GeoCombine
 
     def resources(type)
       return [] if @metadata['resources'].nil?
+
       @metadata['resources'].select { |resource| resource['resource_locator_function'] == type }
     end
 

data/lib/geo_combine/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'thor'
 require 'rake'
 
@@ -17,7 +19,7 @@ module GeoCombine
       Rake::Task['geocombine:pull'].invoke
     end
 
-    desc "index", "Index all of the GeoBlacklight documents"
+    desc 'index', 'Index all of the GeoBlacklight documents'
     def index
       Rake::Task['geocombine:index'].invoke
     end

data/lib/geo_combine/esri_open_data.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   # Data model for ESRI's open data portal metadata
   class EsriOpenData

data/lib/geo_combine/exceptions.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 module GeoCombine
   module Exceptions
     class InvalidDCTReferences < StandardError
     end
+
     class InvalidGeometry < StandardError
     end
   end

data/lib/geo_combine/fgdc.rb

@@ -1,9 +1,9 @@
-module GeoCombine
+# frozen_string_literal: true
 
+module GeoCombine
   ##
   # FIXME: FGDC parsing, transformations are still experimental
   class Fgdc < Metadata
-
     ##
     # Returns a Nokogiri::XSLT object containing the FGDC to GeoBlacklight XSL
     # @return [Nokogiri::XSLT]

data/lib/geo_combine/formats.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   ##
   # Translation dictionary for mime-type to valid GeoBlacklight-Schema formats

data/lib/geo_combine/formatting.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   ##
   # Mixin used for formatting metadata fields
@@ -28,7 +30,7 @@ module GeoCombine
 
     # slugs should be lowercase and only have a-z, A-Z, 0-9, and -
     def sluggify(slug)
-      slug.gsub(/[^a-zA-Z0-9\-]/, '-').gsub(/[\-]+/, '-').downcase
+      slug.gsub(/[^a-zA-Z0-9\-]/, '-').gsub(/-+/, '-').downcase
     end
   end
 end