geo_combine 0.5.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a403eec66c0f81cd06bed4033f411ed137276277e6f9a6eb0a672c64428dcca0
-  data.tar.gz: 7c6e8a61d3d8783a633edac40c57c435a4b6b3d9bd17ff1ae272e794fb6c2b56
+  metadata.gz: 7d47d9cff1e3bf0e3ec2237c65554524b9a624d8c9faf833d1c99c6fdc4f7c94
+  data.tar.gz: 7069057b0b5166f2ed5496af51a270cefdd6ee5aa3937456f10a54b2f5f32536
 SHA512:
-  metadata.gz: b995db35d3452c377d93a9384018e54c01730b76462e5109d3a4ab3e7e24ddcbf3a8fa1b7703b2e571832a054d8a43459a34b18df9b5af418e87107c8509a873
-  data.tar.gz: fc129701c6e2fb138e9f43aab3b2b6e84dc68696813cbb1fb5c7ede89f364c88b8d5717a3e477fc030dfd2ee588a335df0274bf4005f1434c14a841d0f92e0de
+  metadata.gz: 7208f9b13e73b183571861a7d40bcd6a1c1a466b4af7ff6fead7174bdb992a51955ebb986b07094ce195dd2e1d520a3c8b0b51b7c18178b57b3b0eb6db0c6e4e
+  data.tar.gz: 7daf75a1d31036a3659d4e96f261f4f1ce20bfea2083827f32e767ee8f15c14d6c112053cd009ca17cdfc6c433b9f7b64ce524fbf4fdaf7fe56877b1b7c07360

data/.github/workflows/ruby.yml

@@ -0,0 +1,53 @@
+name: CI
+
+on: [push]
+
+jobs:
+  rubocop:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.7
+    - name: Install dependencies
+      run: bundle install
+    - name: Run linter
+      run: bundle exec rubocop
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: [2.7, 3.0, 3.1]
+        faraday_version: [''] # Defaults to whatever's the most recent version.
+        include:
+          - ruby: 2.7
+            faraday_version: '~> 1.0'
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+
+    - name: Install bundler
+      run: gem install bundler -v 2.1.1
+
+    - name: Install dependencies
+      run: bundle _2.1.1_ install
+      env:
+        FARADAY_VERSION: ${{ matrix.faraday_version }}
+
+    - name: Run tests
+      run: bundle exec rake spec
+      env:
+        FARADAY_VERSION: ${{ matrix.faraday_version }}
+
+    - name: Upload coverage artifacts
+      uses: actions/upload-artifact@v2
+      with:
+        name: coverage
+        path: coverage/

data/.gitignore

@@ -12,3 +12,5 @@
 *.o
 *.a
 mkmf.log
+.tool-versions
+.byebug_history

data/.rubocop.yml

@@ -0,0 +1,20 @@
+require:
+  - rubocop-rspec
+  - rubocop-rake
+
+inherit_from: .rubocop_todo.yml
+
+AllCops:
+  TargetRubyVersion: 2.7
+  DisplayCopNames: true
+  NewCops: enable
+  Exclude:
+  - 'geo_combine.gemspec'
+  - 'tmp/**/*'
+
+RSpec/DescribeClass:
+  Enabled: false
+
+RSpec/BeforeAfterAll:
+  Exclude:
+    - 'spec/lib/tasks/geo_combine_spec.rb'

data/.rubocop_todo.yml

@@ -0,0 +1,165 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2022-02-17 18:38:52 UTC using RuboCop version 1.25.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+Lint/RescueException:
+  Exclude:
+    - 'spec/helpers.rb'
+
+# Offense count: 1
+Lint/UselessAssignment:
+  Exclude:
+    - 'spec/helpers.rb'
+
+# Offense count: 7
+# Configuration parameters: IgnoredMethods, CountRepeatedAttributes.
+Metrics/AbcSize:
+  Max: 33
+
+# Offense count: 25
+# Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
+# IgnoredMethods: refine
+Metrics/BlockLength:
+  Max: 181
+
+# Offense count: 1
+# Configuration parameters: CountComments, CountAsOne.
+Metrics/ClassLength:
+  Max: 152
+
+# Offense count: 3
+# Configuration parameters: IgnoredMethods.
+Metrics/CyclomaticComplexity:
+  Max: 11
+
+# Offense count: 10
+# Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
+Metrics/MethodLength:
+  Max: 21
+
+# Offense count: 1
+# Configuration parameters: CountComments, CountAsOne.
+Metrics/ModuleLength:
+  Max: 1657
+
+# Offense count: 1
+# Configuration parameters: IgnoredMethods.
+Metrics/PerceivedComplexity:
+  Max: 11
+
+# Offense count: 9
+# Configuration parameters: Prefixes.
+# Prefixes: when, with, without
+RSpec/ContextWording:
+  Exclude:
+    - 'spec/lib/geo_combine/geoblacklight_spec.rb'
+    - 'spec/lib/geo_combine/ogp_spec.rb'
+
+# Offense count: 9
+# Configuration parameters: CountAsOne.
+RSpec/ExampleLength:
+  Max: 12
+
+# Offense count: 4
+RSpec/ExpectInHook:
+  Exclude:
+    - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
+    - 'spec/lib/geo_combine/geoblacklight_spec.rb'
+
+# Offense count: 1
+# Configuration parameters: Include, CustomTransform, IgnoreMethods, SpecSuffixOnly.
+# Include: **/*_spec*rb*, **/spec/**/*
+RSpec/FilePath:
+  Exclude:
+    - 'spec/lib/geo_combine_spec.rb'
+
+# Configuration parameters: .
+# SupportedStyles: have_received, receive
+RSpec/MessageSpies:
+  EnforcedStyle: have_received
+  Exclude:
+    - 'spec/lib/geo_combine/esri_open_data_spec.rb'
+    - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
+    - 'spec/lib/geo_combine/geoblacklight_spec.rb'
+    - 'spec/lib/geo_combine/ogp_spec.rb'
+    - 'spec/lib/geo_combine_spec.rb'
+
+# Offense count: 39
+RSpec/MultipleExpectations:
+  Max: 5
+
+# Offense count: 3
+# Configuration parameters: AllowSubject.
+RSpec/MultipleMemoizedHelpers:
+  Max: 7
+
+# Offense count: 5
+# Configuration parameters: IgnoreSharedExamples.
+RSpec/NamedSubject:
+  Exclude:
+    - 'spec/lib/geo_combine/formatting_spec.rb'
+
+# Offense count: 8
+RSpec/NestedGroups:
+  Max: 4
+
+# Offense count: 1
+RSpec/OverwritingSetup:
+  Exclude:
+    - 'spec/lib/geo_combine/geoblacklight_spec.rb'
+
+# Offense count: 2
+RSpec/RepeatedExampleGroupBody:
+  Exclude:
+    - 'spec/lib/geo_combine/iso19139_spec.rb'
+
+# Offense count: 19
+RSpec/StubbedMock:
+  Exclude:
+    - 'spec/lib/geo_combine/esri_open_data_spec.rb'
+    - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
+    - 'spec/lib/geo_combine/geoblacklight_spec.rb'
+    - 'spec/lib/geo_combine/ogp_spec.rb'
+    - 'spec/lib/geo_combine_spec.rb'
+
+# Offense count: 5
+RSpec/SubjectStub:
+  Exclude:
+    - 'spec/lib/geo_combine/ogp_spec.rb'
+
+# Offense count: 1
+# Configuration parameters: IgnoreNameless, IgnoreSymbolicNames.
+RSpec/VerifiedDoubles:
+  Exclude:
+    - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
+
+# Offense count: 1
+Security/Open:
+  Exclude:
+    - 'lib/geo_combine/geoblacklight.rb'
+
+# Offense count: 7
+# Configuration parameters: AllowedConstants.
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'lib/geo_combine/bounding_box.rb'
+    - 'lib/geo_combine/ckan_metadata.rb'
+    - 'lib/geo_combine/cli.rb'
+    - 'lib/geo_combine/geo_blacklight_harvester.rb'
+    - 'lib/geo_combine/geoblacklight.rb'
+    - 'lib/geo_combine/geometry_types.rb'
+    - 'lib/geo_combine/iso19139.rb'
+
+# Offense count: 7
+# Cop supports --auto-correct.
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Layout/LineLength:
+  Max: 159

data/Gemfile

@@ -1,7 +1,8 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in geo_combine.gemspec
 gemspec
 
-gem 'coveralls', require: false
 gem 'byebug'

data/README.md

@@ -1,6 +1,8 @@
 # GeoCombine
 
-[![Build Status](https://travis-ci.org/OpenGeoMetadata/GeoCombine.svg?branch=master)](https://travis-ci.org/OpenGeoMetadata/GeoCombine) | [![Coverage Status](https://coveralls.io/repos/OpenGeoMetadata/GeoCombine/badge.svg?branch=master)](https://coveralls.io/r/OpenGeoMetadata/GeoCombine?branch=master)
+   ![CI](https://github.com/OpenGeoMetadata/GeoCombine/actions/workflows/ruby.yml/badge.svg)
+| [![Coverage Status](https://img.shields.io/badge/coverage-95%25-brightgreen)]()
+| [![Gem Version](https://img.shields.io/gem/v/geo_combine.svg)](https://github.com/OpenGeoMetadata/GeoCombine/releases)
 
 
 A Ruby toolkit for managing geospatial metadata, including:
@@ -61,6 +63,12 @@ You can also specify a single repository:
 $ bundle exec rake geocombine:clone[edu.stanford.purl]
 ```
 
+*Note: If you are using zsh, you will need to use escape characters in front of the brackets:*
+
+```sh
+$ bundle exec rake geocombine:clone\[edu.stanford.purl\]
+```
+
 #### Update local OpenGeoMetadata repositories
 
 ```sh
@@ -75,6 +83,12 @@ You can also specify a single repository:
 $ bundle exec rake geocombine:pull[edu.stanford.purl]
 ```
 
+*Note: If you are using zsh, you will need to use escape characters in front of the brackets:*
+
+```sh
+$ bundle exec rake geocombine:pull\[edu.stanford.purl\]
+```
+
 #### Index GeoBlacklight documents
 
 To index into Solr, GeoCombine requires a Solr instance that is running the
@@ -105,12 +119,16 @@ $ SOLR_COMMIT_WITHIN=100 bundle exec rake geocombine:index
 
 GeoCombine provides a Harvester class and rake task to harvest and index content from GeoBlacklight sites (or any site that follows the Blacklight API format). Given that the configurations can change from consumer to consumer and site to site, the class provides a relatively simple configuration API. This can be configured in an initializer, a wrapping rake task, or any other ruby context where the rake task our class would be invoked.
 
+```sh
+bundle exec rake geocombine:geoblacklight_harvester:index[YOUR_CONFIGURED_SITE_KEY]
+```
+
 #### Harvester configuration
 
 Only the sites themselves are required to be configured but there are various configuration options that can (optionally) be supplied to modify the harvester's behavior.
 
 ```ruby
-GeoCombine::GeoBlacklightIndexer.configure do
+GeoCombine::GeoBlacklightHarvester.configure do
   {
     commit_within: '10000',
     crawl_delay: 1, # All sites

data/Rakefile

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 require 'bundler/gem_tasks'
 
-Dir.glob('lib/tasks/*.rake').each { |r| load r}
+Dir.glob('lib/tasks/*.rake').each { |r| load r }
 
 desc 'Run console for development'
 task :console do
@@ -17,7 +19,7 @@ begin
 
   RSpec::Core::RakeTask.new(:spec)
 
-  task :default => :spec
+  task default: :spec
 rescue LoadError
   # no rspec available
 end

data/bin/geocombine

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'geo_combine/cli'
 

data/geo_combine.gemspec

@@ -20,14 +20,19 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'activesupport'
   spec.add_dependency 'rsolr'
-  spec.add_dependency 'net-http-persistent', '~> 2.0' # pin since faraday (rsolr) doesn't work correctly with 3.x
   spec.add_dependency 'nokogiri'
   spec.add_dependency 'json-schema'
   spec.add_dependency 'sanitize'
   spec.add_dependency 'thor'
+  spec.add_dependency 'faraday-net_http_persistent', '~> 2.0'
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rspec-html-matchers'
+  spec.add_development_dependency 'rubocop', '~> 1.25'
+  spec.add_development_dependency 'rubocop-rspec', '~> 2.8'
+  spec.add_development_dependency 'rubocop-rake'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'webmock', '~> 3.14'
 end

data/lib/geo_combine/bounding_box.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   class BoundingBox
     attr_reader :west, :south, :east, :north
@@ -24,11 +26,13 @@ module GeoCombine
     def valid?
       [south, north].map do |coord|
         next if (-90..90).cover?(coord)
+
         raise GeoCombine::Exceptions::InvalidGeometry,
               "#{coord} should be in range -90 90"
       end
       [east, west].map do |coord|
         next if (-180..180).cover?(coord)
+
         raise GeoCombine::Exceptions::InvalidGeometry,
               "#{coord} should be in range -180 180"
       end
@@ -45,7 +49,8 @@ module GeoCombine
 
     def self.from_envelope(envelope)
       return if envelope.nil?
-      envelope = envelope[/.*ENVELOPE\(([^\)]*)/, 1].split(',')
+
+      envelope = envelope[/.*ENVELOPE\(([^)]*)/, 1].split(',')
       new(
         west: envelope[0],
         south: envelope[3],
@@ -59,6 +64,7 @@ module GeoCombine
     # @param [String] delimiter "," or " "
     def self.from_string_delimiter(spatial, delimiter: ',')
       return if spatial.nil?
+
       spatial = spatial.split(delimiter)
       new(
         west: spatial[0],

data/lib/geo_combine/ckan_metadata.rb

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 module GeoCombine
   class CkanMetadata
-    MAX_STRING_LENGTH = 32765 # Solr limit
+    MAX_STRING_LENGTH = 32_765 # Solr limit
 
     attr_reader :metadata
+
     def initialize(metadata)
       @metadata = JSON.parse(metadata)
     end
@@ -31,7 +34,7 @@ module GeoCombine
         dc_subject_sm: subjects,
         dct_references_s: external_references.to_json.to_s,
         dc_format_s: downloadable? ? 'ZIP' : nil # TODO: we only allow direct ZIP file downloads
-      }.select { |_k, v| !v.nil? }
+      }.compact
     end
 
     def organization
@@ -54,7 +57,7 @@ module GeoCombine
       begin
         return bbox.to_envelope if bbox.valid?
       rescue GeoCombine::Exceptions::InvalidGeometry
-        return nil
+        nil
       end
     end
 
@@ -66,7 +69,7 @@ module GeoCombine
       begin
         return bbox.to_envelope if bbox.valid?
       rescue GeoCombine::Exceptions::InvalidGeometry
-        return nil
+        nil
       end
     end
 
@@ -87,11 +90,9 @@ module GeoCombine
         'http://schema.org/url' => resource_urls('information').first
       }
 
-      if downloadable?
-        h['http://schema.org/downloadUrl'] = resource_urls('download').first
-      end
+      h['http://schema.org/downloadUrl'] = resource_urls('download').first if downloadable?
 
-      h.select { |_k, v| !v.nil? }
+      h.compact
     end
 
     def downloadable?
@@ -100,6 +101,7 @@ module GeoCombine
 
     def resources(type)
       return [] if @metadata['resources'].nil?
+
       @metadata['resources'].select { |resource| resource['resource_locator_function'] == type }
     end
 

data/lib/geo_combine/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'thor'
 require 'rake'
 
@@ -17,7 +19,7 @@ module GeoCombine
       Rake::Task['geocombine:pull'].invoke
     end
 
-    desc "index", "Index all of the GeoBlacklight documents"
+    desc 'index', 'Index all of the GeoBlacklight documents'
     def index
       Rake::Task['geocombine:index'].invoke
     end

data/lib/geo_combine/esri_open_data.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   # Data model for ESRI's open data portal metadata
   class EsriOpenData

data/lib/geo_combine/exceptions.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 module GeoCombine
   module Exceptions
     class InvalidDCTReferences < StandardError
     end
+
     class InvalidGeometry < StandardError
     end
   end

data/lib/geo_combine/fgdc.rb

@@ -1,9 +1,9 @@
-module GeoCombine
+# frozen_string_literal: true
 
+module GeoCombine
   ##
   # FIXME: FGDC parsing, transformations are still experimental
   class Fgdc < Metadata
-
     ##
     # Returns a Nokogiri::XSLT object containing the FGDC to GeoBlacklight XSL
     # @return [Nokogiri::XSLT]

data/lib/geo_combine/formats.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   ##
   # Translation dictionary for mime-type to valid GeoBlacklight-Schema formats

data/lib/geo_combine/formatting.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   ##
   # Mixin used for formatting metadata fields
@@ -28,7 +30,7 @@ module GeoCombine
 
     # slugs should be lowercase and only have a-z, A-Z, 0-9, and -
     def sluggify(slug)
-      slug.gsub(/[^a-zA-Z0-9\-]/, '-').gsub(/[\-]+/, '-').downcase
+      slug.gsub(/[^a-zA-Z0-9\-]/, '-').gsub(/-+/, '-').downcase
     end
   end
 end

data/lib/geo_combine/geo_blacklight_harvester.rb

@@ -30,17 +30,21 @@ module GeoCombine
       end
 
       def document_transformer
-        @document_transformer || ->(document) do
+        @document_transformer || lambda do |document|
           document.delete('_version_')
           document.delete('score')
           document.delete('timestamp')
+          document.delete('solr_bboxtype__minX')
+          document.delete('solr_bboxtype__minY')
+          document.delete('solr_bboxtype__maxX')
+          document.delete('solr_bboxtype__maxY')
           document
         end
       end
     end
 
-
     attr_reader :site, :site_key
+
     def initialize(site_key)
       @site_key = site_key
       @site = self.class.config[site_key]
@@ -55,7 +59,7 @@ module GeoCombine
 
       response_class.new(response: response, base_url: base_url).documents.each do |docs|
         docs.map! do |document|
-          self.class.document_transformer.call(document) if self.class.document_transformer
+          self.class.document_transformer&.call(document)
         end.compact
 
         puts "Adding #{docs.count} documents to solr" if self.class.config[:debug]
@@ -77,7 +81,8 @@ module GeoCombine
         elsif keys.any? && %w[links data].all? { |param| keys.include?(param) }
           ModernBlacklightResponse
         else
-          raise NotImplementedError, "The following json response was not able to be parsed by the GeoBlacklightHarvester\n#{json}"
+          raise NotImplementedError,
+                "The following json response was not able to be parsed by the GeoBlacklightHarvester\n#{json}"
         end
       end
     end
@@ -85,6 +90,7 @@ module GeoCombine
     class LegacyBlacklightResponse
       attr_reader :base_url
       attr_accessor :response, :page
+
       def initialize(response:, base_url:)
         @base_url = base_url
         @response = response
@@ -94,16 +100,17 @@ module GeoCombine
       def documents
         return enum_for(:documents) unless block_given?
 
-        while current_page && total_pages && (current_page <= total_pages) do
+        while current_page && total_pages && (current_page <= total_pages)
           yield response.dig('response', 'docs')
 
           break if current_page == total_pages
+
           self.page += 1
           puts "Fetching page #{page} @ #{url}" if GeoCombine::GeoBlacklightHarvester.config[:debug]
 
           begin
             self.response = JSON.parse(Net::HTTP.get(URI(url)))
-          rescue => e
+          rescue StandardError => e
             puts "Request for #{url} failed with #{e}"
             self.response = nil
           end
@@ -130,6 +137,7 @@ module GeoCombine
     class ModernBlacklightResponse
       attr_reader :base_url
       attr_accessor :response, :page
+
       def initialize(response:, base_url:)
         @base_url = base_url
         @response = response
@@ -146,11 +154,13 @@ module GeoCombine
 
           url = response.dig('links', 'next')
           break unless url
+
+          url = "#{url}&format=json"
           self.page += 1
           puts "Fetching page #{page} @ #{url}" if GeoCombine::GeoBlacklightHarvester.config[:debug]
           begin
             self.response = JSON.parse(Net::HTTP.get(URI(url)))
-          rescue => e
+          rescue StandardError => e
             puts "Request for #{url} failed with #{e}"
             self.response = nil
           end
@@ -162,13 +172,11 @@ module GeoCombine
       def documents_from_urls(urls)
         puts "Fetching #{urls.count} documents for page #{page}" if GeoCombine::GeoBlacklightHarvester.config[:debug]
         urls.map do |url|
-          begin
-            JSON.parse(Net::HTTP.get(URI("#{url}/raw")))
-          rescue => e
-            puts "Fetching \"#{url}/raw\" failed with #{e}"
+          JSON.parse(Net::HTTP.get(URI("#{url}/raw")))
+        rescue StandardError => e
+          puts "Fetching \"#{url}/raw\" failed with #{e}"
 
-            nil
-          end
+          nil
         end.compact
       end
     end