geo_combine 0.5.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0439618d4248a21c1f4efe91e863d01ad9b97f0798b7f855b60c15eab7457bbf'
-  data.tar.gz: a01ec4dc01b9c6d3dd39e34c178f2271b83d8c51ecf16226e8595a36ddbf37ea
+  metadata.gz: ba7bc8217102aecc80fc75e0ca8915292b0be85c18bac51a2f19b486d33273a0
+  data.tar.gz: f66c6e54dedf27a4e8786e2bbe1aa449d1aa5c27e1f5fe8978b7dd75206fc496
 SHA512:
-  metadata.gz: 4c7c77f1cf1b60e7438f144d97a1af16126cdc1c9567ae2006cdcaa26c47fc0d5af8672e6368b5b139789b8f44f55cbadc1bd1f220b84b4a36c4bfc78dba3a65
-  data.tar.gz: d33c6c00ed6b39a6c2a8238ad9e14304bb59e450a031b8b8836635bb71d4a2deed3b8b0de68ca10c6e0ae9e9013f67acab42e765534475e2040af76f65a7a1ad
+  metadata.gz: 5d191f0f24795e8544a9b231116fedb4ba635ed3dba333de66f60ccaca5776fdaddb3d399be25a77112cac370d1f2eab36ab1819a7216eb5175678eb4b9aae29
+  data.tar.gz: 0d45aec320e4c543c349608d36cfb632197fedb8571adabd77ab8e11d216c1922a5dd135862b3de51cbce70a3ab32089c67561f4bd9c16458eeb864347e46011

data/.github/workflows/ruby.yml

@@ -0,0 +1,53 @@
+name: CI
+
+on: [push]
+
+jobs:
+  rubocop:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.7
+    - name: Install dependencies
+      run: bundle install
+    - name: Run linter
+      run: bundle exec rubocop
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: [2.7, 3.0, 3.1]
+        faraday_version: [''] # Defaults to whatever's the most recent version.
+        include:
+          - ruby: 2.7
+            faraday_version: '~> 1.0'
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+
+    - name: Install bundler
+      run: gem install bundler -v 2.1.1
+
+    - name: Install dependencies
+      run: bundle _2.1.1_ install
+      env:
+        FARADAY_VERSION: ${{ matrix.faraday_version }}
+
+    - name: Run tests
+      run: bundle exec rake spec
+      env:
+        FARADAY_VERSION: ${{ matrix.faraday_version }}
+
+    - name: Upload coverage artifacts
+      uses: actions/upload-artifact@v2
+      with:
+        name: coverage
+        path: coverage/

data/.gitignore

@@ -12,3 +12,5 @@
 *.o
 *.a
 mkmf.log
+.tool-versions
+.byebug_history

data/.rubocop.yml

@@ -0,0 +1,20 @@
+require:
+  - rubocop-rspec
+  - rubocop-rake
+
+inherit_from: .rubocop_todo.yml
+
+AllCops:
+  TargetRubyVersion: 2.7
+  DisplayCopNames: true
+  NewCops: enable
+  Exclude:
+  - 'geo_combine.gemspec'
+  - 'tmp/**/*'
+
+RSpec/DescribeClass:
+  Enabled: false
+
+RSpec/BeforeAfterAll:
+  Exclude:
+    - 'spec/lib/tasks/geo_combine_spec.rb'

data/.rubocop_todo.yml

@@ -0,0 +1,165 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2022-02-17 18:38:52 UTC using RuboCop version 1.25.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+Lint/RescueException:
+  Exclude:
+    - 'spec/helpers.rb'
+
+# Offense count: 1
+Lint/UselessAssignment:
+  Exclude:
+    - 'spec/helpers.rb'
+
+# Offense count: 7
+# Configuration parameters: IgnoredMethods, CountRepeatedAttributes.
+Metrics/AbcSize:
+  Max: 33
+
+# Offense count: 25
+# Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
+# IgnoredMethods: refine
+Metrics/BlockLength:
+  Max: 181
+
+# Offense count: 1
+# Configuration parameters: CountComments, CountAsOne.
+Metrics/ClassLength:
+  Max: 152
+
+# Offense count: 3
+# Configuration parameters: IgnoredMethods.
+Metrics/CyclomaticComplexity:
+  Max: 11
+
+# Offense count: 10
+# Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
+Metrics/MethodLength:
+  Max: 21
+
+# Offense count: 1
+# Configuration parameters: CountComments, CountAsOne.
+Metrics/ModuleLength:
+  Max: 1657
+
+# Offense count: 1
+# Configuration parameters: IgnoredMethods.
+Metrics/PerceivedComplexity:
+  Max: 11
+
+# Offense count: 9
+# Configuration parameters: Prefixes.
+# Prefixes: when, with, without
+RSpec/ContextWording:
+  Exclude:
+    - 'spec/lib/geo_combine/geoblacklight_spec.rb'
+    - 'spec/lib/geo_combine/ogp_spec.rb'
+
+# Offense count: 9
+# Configuration parameters: CountAsOne.
+RSpec/ExampleLength:
+  Max: 12
+
+# Offense count: 4
+RSpec/ExpectInHook:
+  Exclude:
+    - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
+    - 'spec/lib/geo_combine/geoblacklight_spec.rb'
+
+# Offense count: 1
+# Configuration parameters: Include, CustomTransform, IgnoreMethods, SpecSuffixOnly.
+# Include: **/*_spec*rb*, **/spec/**/*
+RSpec/FilePath:
+  Exclude:
+    - 'spec/lib/geo_combine_spec.rb'
+
+# Configuration parameters: .
+# SupportedStyles: have_received, receive
+RSpec/MessageSpies:
+  EnforcedStyle: have_received
+  Exclude:
+    - 'spec/lib/geo_combine/esri_open_data_spec.rb'
+    - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
+    - 'spec/lib/geo_combine/geoblacklight_spec.rb'
+    - 'spec/lib/geo_combine/ogp_spec.rb'
+    - 'spec/lib/geo_combine_spec.rb'
+
+# Offense count: 39
+RSpec/MultipleExpectations:
+  Max: 5
+
+# Offense count: 3
+# Configuration parameters: AllowSubject.
+RSpec/MultipleMemoizedHelpers:
+  Max: 7
+
+# Offense count: 5
+# Configuration parameters: IgnoreSharedExamples.
+RSpec/NamedSubject:
+  Exclude:
+    - 'spec/lib/geo_combine/formatting_spec.rb'
+
+# Offense count: 8
+RSpec/NestedGroups:
+  Max: 4
+
+# Offense count: 1
+RSpec/OverwritingSetup:
+  Exclude:
+    - 'spec/lib/geo_combine/geoblacklight_spec.rb'
+
+# Offense count: 2
+RSpec/RepeatedExampleGroupBody:
+  Exclude:
+    - 'spec/lib/geo_combine/iso19139_spec.rb'
+
+# Offense count: 19
+RSpec/StubbedMock:
+  Exclude:
+    - 'spec/lib/geo_combine/esri_open_data_spec.rb'
+    - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
+    - 'spec/lib/geo_combine/geoblacklight_spec.rb'
+    - 'spec/lib/geo_combine/ogp_spec.rb'
+    - 'spec/lib/geo_combine_spec.rb'
+
+# Offense count: 5
+RSpec/SubjectStub:
+  Exclude:
+    - 'spec/lib/geo_combine/ogp_spec.rb'
+
+# Offense count: 1
+# Configuration parameters: IgnoreNameless, IgnoreSymbolicNames.
+RSpec/VerifiedDoubles:
+  Exclude:
+    - 'spec/lib/geo_combine/geo_blacklight_harvester_spec.rb'
+
+# Offense count: 1
+Security/Open:
+  Exclude:
+    - 'lib/geo_combine/geoblacklight.rb'
+
+# Offense count: 7
+# Configuration parameters: AllowedConstants.
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'lib/geo_combine/bounding_box.rb'
+    - 'lib/geo_combine/ckan_metadata.rb'
+    - 'lib/geo_combine/cli.rb'
+    - 'lib/geo_combine/geo_blacklight_harvester.rb'
+    - 'lib/geo_combine/geoblacklight.rb'
+    - 'lib/geo_combine/geometry_types.rb'
+    - 'lib/geo_combine/iso19139.rb'
+
+# Offense count: 7
+# Cop supports --auto-correct.
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Layout/LineLength:
+  Max: 159

data/Gemfile

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in geo_combine.gemspec
 gemspec
 
-gem 'coveralls', require: false
 gem 'byebug'
+gem 'faraday', ENV['FARADAY_VERSION'] if ENV['FARADAY_VERSION'].to_s != ''

data/README.md

@@ -1,6 +1,8 @@
 # GeoCombine
 
-[![Build Status](https://travis-ci.org/OpenGeoMetadata/GeoCombine.svg?branch=master)](https://travis-ci.org/OpenGeoMetadata/GeoCombine) | [![Coverage Status](https://coveralls.io/repos/OpenGeoMetadata/GeoCombine/badge.svg?branch=master)](https://coveralls.io/r/OpenGeoMetadata/GeoCombine?branch=master)
+   ![CI](https://github.com/OpenGeoMetadata/GeoCombine/actions/workflows/ruby.yml/badge.svg)
+| [![Coverage Status](https://img.shields.io/badge/coverage-95%25-brightgreen)]()
+| [![Gem Version](https://img.shields.io/gem/v/geo_combine.svg)](https://github.com/OpenGeoMetadata/GeoCombine/releases)
 
 
 A Ruby toolkit for managing geospatial metadata, including:
@@ -61,6 +63,12 @@ You can also specify a single repository:
 $ bundle exec rake geocombine:clone[edu.stanford.purl]
 ```
 
+*Note: If you are using zsh, you will need to use escape characters in front of the brackets:*
+
+```sh
+$ bundle exec rake geocombine:clone\[edu.stanford.purl\]
+```
+
 #### Update local OpenGeoMetadata repositories
 
 ```sh
@@ -75,6 +83,12 @@ You can also specify a single repository:
 $ bundle exec rake geocombine:pull[edu.stanford.purl]
 ```
 
+*Note: If you are using zsh, you will need to use escape characters in front of the brackets:*
+
+```sh
+$ bundle exec rake geocombine:pull\[edu.stanford.purl\]
+```
+
 #### Index GeoBlacklight documents
 
 To index into Solr, GeoCombine requires a Solr instance that is running the

data/Rakefile

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 require 'bundler/gem_tasks'
 
-Dir.glob('lib/tasks/*.rake').each { |r| load r}
+Dir.glob('lib/tasks/*.rake').each { |r| load r }
 
 desc 'Run console for development'
 task :console do
@@ -17,7 +19,7 @@ begin
 
   RSpec::Core::RakeTask.new(:spec)
 
-  task :default => :spec
+  task default: :spec
 rescue LoadError
   # no rspec available
 end

data/bin/geocombine

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'geo_combine/cli'
 

data/geo_combine.gemspec

@@ -30,4 +30,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rspec-html-matchers'
+  spec.add_development_dependency 'rubocop', '~> 1.25'
+  spec.add_development_dependency 'rubocop-rspec', '~> 2.8'
+  spec.add_development_dependency 'rubocop-rake'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'webmock', '~> 3.14'
 end

data/lib/geo_combine/bounding_box.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   class BoundingBox
     attr_reader :west, :south, :east, :north
@@ -24,11 +26,13 @@ module GeoCombine
     def valid?
       [south, north].map do |coord|
         next if (-90..90).cover?(coord)
+
         raise GeoCombine::Exceptions::InvalidGeometry,
               "#{coord} should be in range -90 90"
       end
       [east, west].map do |coord|
         next if (-180..180).cover?(coord)
+
         raise GeoCombine::Exceptions::InvalidGeometry,
               "#{coord} should be in range -180 180"
       end
@@ -45,7 +49,8 @@ module GeoCombine
 
     def self.from_envelope(envelope)
       return if envelope.nil?
-      envelope = envelope[/.*ENVELOPE\(([^\)]*)/, 1].split(',')
+
+      envelope = envelope[/.*ENVELOPE\(([^)]*)/, 1].split(',')
       new(
         west: envelope[0],
         south: envelope[3],
@@ -59,6 +64,7 @@ module GeoCombine
     # @param [String] delimiter "," or " "
     def self.from_string_delimiter(spatial, delimiter: ',')
       return if spatial.nil?
+
       spatial = spatial.split(delimiter)
       new(
         west: spatial[0],

data/lib/geo_combine/ckan_metadata.rb

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 module GeoCombine
   class CkanMetadata
-    MAX_STRING_LENGTH = 32765 # Solr limit
+    MAX_STRING_LENGTH = 32_765 # Solr limit
 
     attr_reader :metadata
+
     def initialize(metadata)
       @metadata = JSON.parse(metadata)
     end
@@ -31,7 +34,7 @@ module GeoCombine
         dc_subject_sm: subjects,
         dct_references_s: external_references.to_json.to_s,
         dc_format_s: downloadable? ? 'ZIP' : nil # TODO: we only allow direct ZIP file downloads
-      }.select { |_k, v| !v.nil? }
+      }.compact
     end
 
     def organization
@@ -54,7 +57,7 @@ module GeoCombine
       begin
         return bbox.to_envelope if bbox.valid?
       rescue GeoCombine::Exceptions::InvalidGeometry
-        return nil
+        nil
       end
     end
 
@@ -66,7 +69,7 @@ module GeoCombine
       begin
         return bbox.to_envelope if bbox.valid?
       rescue GeoCombine::Exceptions::InvalidGeometry
-        return nil
+        nil
       end
     end
 
@@ -87,11 +90,9 @@ module GeoCombine
         'http://schema.org/url' => resource_urls('information').first
       }
 
-      if downloadable?
-        h['http://schema.org/downloadUrl'] = resource_urls('download').first
-      end
+      h['http://schema.org/downloadUrl'] = resource_urls('download').first if downloadable?
 
-      h.select { |_k, v| !v.nil? }
+      h.compact
     end
 
     def downloadable?
@@ -100,6 +101,7 @@ module GeoCombine
 
     def resources(type)
       return [] if @metadata['resources'].nil?
+
       @metadata['resources'].select { |resource| resource['resource_locator_function'] == type }
     end
 

data/lib/geo_combine/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'thor'
 require 'rake'
 
@@ -17,7 +19,7 @@ module GeoCombine
       Rake::Task['geocombine:pull'].invoke
     end
 
-    desc "index", "Index all of the GeoBlacklight documents"
+    desc 'index', 'Index all of the GeoBlacklight documents'
     def index
       Rake::Task['geocombine:index'].invoke
     end

data/lib/geo_combine/esri_open_data.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   # Data model for ESRI's open data portal metadata
   class EsriOpenData

data/lib/geo_combine/exceptions.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+
 module GeoCombine
   module Exceptions
     class InvalidDCTReferences < StandardError
     end
+
     class InvalidGeometry < StandardError
     end
   end

data/lib/geo_combine/fgdc.rb

@@ -1,9 +1,9 @@
-module GeoCombine
+# frozen_string_literal: true
 
+module GeoCombine
   ##
   # FIXME: FGDC parsing, transformations are still experimental
   class Fgdc < Metadata
-
     ##
     # Returns a Nokogiri::XSLT object containing the FGDC to GeoBlacklight XSL
     # @return [Nokogiri::XSLT]

data/lib/geo_combine/formats.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   ##
   # Translation dictionary for mime-type to valid GeoBlacklight-Schema formats

data/lib/geo_combine/formatting.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module GeoCombine
   ##
   # Mixin used for formatting metadata fields
@@ -28,7 +30,7 @@ module GeoCombine
 
     # slugs should be lowercase and only have a-z, A-Z, 0-9, and -
     def sluggify(slug)
-      slug.gsub(/[^a-zA-Z0-9\-]/, '-').gsub(/[\-]+/, '-').downcase
+      slug.gsub(/[^a-zA-Z0-9\-]/, '-').gsub(/-+/, '-').downcase
     end
   end
 end

data/lib/geo_combine/geo_blacklight_harvester.rb

@@ -30,17 +30,21 @@ module GeoCombine
       end
 
       def document_transformer
-        @document_transformer || ->(document) do
+        @document_transformer || lambda do |document|
           document.delete('_version_')
           document.delete('score')
           document.delete('timestamp')
+          document.delete('solr_bboxtype__minX')
+          document.delete('solr_bboxtype__minY')
+          document.delete('solr_bboxtype__maxX')
+          document.delete('solr_bboxtype__maxY')
           document
         end
       end
     end
 
-
     attr_reader :site, :site_key
+
     def initialize(site_key)
       @site_key = site_key
       @site = self.class.config[site_key]
@@ -55,7 +59,7 @@ module GeoCombine
 
       response_class.new(response: response, base_url: base_url).documents.each do |docs|
         docs.map! do |document|
-          self.class.document_transformer.call(document) if self.class.document_transformer
+          self.class.document_transformer&.call(document)
         end.compact
 
         puts "Adding #{docs.count} documents to solr" if self.class.config[:debug]
@@ -77,7 +81,8 @@ module GeoCombine
         elsif keys.any? && %w[links data].all? { |param| keys.include?(param) }
           ModernBlacklightResponse
         else
-          raise NotImplementedError, "The following json response was not able to be parsed by the GeoBlacklightHarvester\n#{json}"
+          raise NotImplementedError,
+                "The following json response was not able to be parsed by the GeoBlacklightHarvester\n#{json}"
         end
       end
     end
@@ -85,6 +90,7 @@ module GeoCombine
     class LegacyBlacklightResponse
       attr_reader :base_url
       attr_accessor :response, :page
+
       def initialize(response:, base_url:)
         @base_url = base_url
         @response = response
@@ -94,16 +100,17 @@ module GeoCombine
       def documents
         return enum_for(:documents) unless block_given?
 
-        while current_page && total_pages && (current_page <= total_pages) do
+        while current_page && total_pages && (current_page <= total_pages)
           yield response.dig('response', 'docs')
 
           break if current_page == total_pages
+
           self.page += 1
           puts "Fetching page #{page} @ #{url}" if GeoCombine::GeoBlacklightHarvester.config[:debug]
 
           begin
             self.response = JSON.parse(Net::HTTP.get(URI(url)))
-          rescue => e
+          rescue StandardError => e
             puts "Request for #{url} failed with #{e}"
             self.response = nil
           end
@@ -130,6 +137,7 @@ module GeoCombine
     class ModernBlacklightResponse
       attr_reader :base_url
       attr_accessor :response, :page
+
       def initialize(response:, base_url:)
         @base_url = base_url
         @response = response
@@ -146,12 +154,13 @@ module GeoCombine
 
           url = response.dig('links', 'next')
           break unless url
+
           url = "#{url}&format=json"
           self.page += 1
           puts "Fetching page #{page} @ #{url}" if GeoCombine::GeoBlacklightHarvester.config[:debug]
           begin
             self.response = JSON.parse(Net::HTTP.get(URI(url)))
-          rescue => e
+          rescue StandardError => e
             puts "Request for #{url} failed with #{e}"
             self.response = nil
           end
@@ -163,13 +172,11 @@ module GeoCombine
       def documents_from_urls(urls)
         puts "Fetching #{urls.count} documents for page #{page}" if GeoCombine::GeoBlacklightHarvester.config[:debug]
         urls.map do |url|
-          begin
-            JSON.parse(Net::HTTP.get(URI("#{url}/raw")))
-          rescue => e
-            puts "Fetching \"#{url}/raw\" failed with #{e}"
+          JSON.parse(Net::HTTP.get(URI("#{url}/raw")))
+        rescue StandardError => e
+          puts "Fetching \"#{url}/raw\" failed with #{e}"
 
-            nil
-          end
+          nil
         end.compact
       end
     end