search_solr_tools 6.2.0 → 6.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: afbece000e765162dcb622479c3b9da58b534e4609b83ccb194a719ae5bb9c03
-  data.tar.gz: 012cb04bdced985254701d6f50fc99340c909fcff8000263c184dd9519282bbf
+  metadata.gz: 310dba4a749b285756284729fe320c2344ac345c3251225dc1a46cc4c0f38d79
+  data.tar.gz: 98a397d862c73f27b9554560c55f2132f6d3211018c4cbe76a98f501cac2d5d0
 SHA512:
-  metadata.gz: 9b0c53d6ff652e840a8014f1ee0e31db62ad5faba634cd7aebf66ac90cd9a6865953ed081169f4b52ca96bf3ac135954a26d8aa6e5e7aa2bc1cf3298f946f260
-  data.tar.gz: 3975437ccf9540013f5c74e71a8e6f676273aab0e4b8a9a78df853b0010120b660f06a7a6ae630d6b40c6c6b8e56fc01b7562a6e2127508218c716e5ff4b41fc
+  metadata.gz: d02152bd10e9bc3ea59d6a2a8448ed2ab63da49db35dc0b23dbf27236f85253987169eaefbe2a3da3e527f9d63d11bdab08d58b3ebb9673579daebbdb8de6097
+  data.tar.gz: 3cc3e2199432b47dd02a1e45238c8405821673c808cd83392ebda9837890f949b193f9267b3afbd760f8449490ef5e97a911ad4c97dd856081307ba7f73095b4

data/CHANGELOG.md

@@ -1,12 +1,23 @@
-## v6.2.0 (2022-07-18)
+## v6.4.0 (2023-08-14)
+
+  - Fixed a bug with the sanitization, which was trying to modify the
+    string directly (causing problems with frozen strings).  Changed to
+    return a new, sanitized string.
+
+## v6.3.0 (2023-07-24)
+
+  - Update Rubocop configuration to actually run against files, and make
+    necessary corrections to comply with Rubocop styling.
+
+## v6.2.0 (2023-07-18)
 
   - Remove deprecated harvesters and associated tests, helpers, etc.
 
-## v6.1.0 (2022-07-14)
+## v6.1.0 (2023-07-14)
 
   - Updated a few other dependencies that weren't at the newest versions.
 
-## v6.0.0 (2022-07-14)
+## v6.0.0 (2023-07-14)
 
   - Updated Ruby to 3.2.2, updated gem dependencies to more recent versions.
 

data/README.md

@@ -210,6 +210,5 @@ license is included in the file COPYING.
 
 Andy Grauch, Brendan Billingsley, Chris Chalstrom, Danielle Harper, Ian
 Truslove, Jonathan Kovarik, Luis Lopez, Miao Liu, Michael Brandt, Stuart Reed,
-Julia Collins, Scott Lewis
-(2023): Arctic Data Explorer SOLR Search software tools. The National Snow and
-Ice Data Center. Software.
+Julia Collins, Scott Lewis (2023): Arctic Data Explorer SOLR Search software tools.
+The National Snow and Ice Data Center. Software. https://doi.org/10.7265/n5jq0xzm

data/bin/search_solr_tools

@@ -47,7 +47,7 @@ class SolrHarvestCLI < Thor
     end
 
     ping_status = SearchSolrTools::Helpers::HarvestStatus.new(
-      SearchSolrTools::Helpers::HarvestStatus::PING_SOLR => solr_success,
+      SearchSolrTools::Helpers::HarvestStatus::PING_SOLR   => solr_success,
       SearchSolrTools::Helpers::HarvestStatus::PING_SOURCE => source_success
     )
     raise SearchSolrTools::Errors::HarvestError, ping_status unless ping_status.ok?
@@ -64,9 +64,9 @@ class SolrHarvestCLI < Thor
       puts "Target: #{target}"
       begin
         harvest_class = get_harvester_class(target)
-        harvester = harvest_class.new(options[:environment], die_on_failure)
+        harvester = harvest_class.new(options[:environment], die_on_failure:)
         ping_status = SearchSolrTools::Helpers::HarvestStatus.new(
-          SearchSolrTools::Helpers::HarvestStatus::PING_SOLR => harvester.ping_solr,
+          SearchSolrTools::Helpers::HarvestStatus::PING_SOLR   => harvester.ping_solr,
           SearchSolrTools::Helpers::HarvestStatus::PING_SOURCE => harvester.ping_source
         )
         raise SearchSolrTools::Errors::HarvestError, ping_status unless ping_status.ok?
@@ -120,7 +120,7 @@ class SolrHarvestCLI < Thor
   no_tasks do
     def harvester_map
       {
-        'nsidc' => SearchSolrTools::Harvesters::NsidcJson,
+        'nsidc'              => SearchSolrTools::Harvesters::NsidcJson,
         'nsidc_auto_suggest' => SearchSolrTools::Harvesters::NsidcAutoSuggest
       }
     end

data/lib/search_solr_tools/config/environments.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 require 'yaml'
 
 module SearchSolrTools
   # configuration to work with solr locally, or on integration/qa/staging/prod
   module SolrEnvironments
-    YAML_ENVS = YAML.load_file(File.expand_path('../environments.yaml', __FILE__))
+    YAML_ENVS = YAML.load_file(File.expand_path('environments.yaml', __dir__))
 
     def self.[](env = :development)
       YAML_ENVS[:common].merge(YAML_ENVS[env.to_sym])

data/lib/search_solr_tools/errors/harvest_error.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SearchSolrTools
   module Errors
     class HarvestError < StandardError
@@ -10,34 +12,47 @@ module SearchSolrTools
       ERRCODE_OTHER = 128
 
       ERRCODE_DESC = {
-          ERRCODE_SOLR_PING => 'Solr instance did not return a successful ping',
-          ERRCODE_SOURCE_PING => 'Source to be harvested did not return a successful ping',
-          ERRCODE_SOURCE_NO_RESULTS => 'Source to be harvested returned no documents matching query',
-          ERRCODE_SOURCE_HARVEST_ERROR => 'One or more source documents returned an error when trying to retrieve or translate',
-          ERRCODE_DOCUMENT_INVALID => 'One or more documents to be harvested was invalid (malformed)',
-          ERRCODE_INGEST_ERROR => 'Solr returned an error trying to ingest one or more harvested documents',
-          ERRCODE_OTHER => 'General error code for non-harvest related issues'
+        ERRCODE_SOLR_PING            => 'Solr instance did not return a successful ping',
+        ERRCODE_SOURCE_PING          => 'Source to be harvested did not return a successful ping',
+        ERRCODE_SOURCE_NO_RESULTS    => 'Source to be harvested returned no documents matching query',
+        ERRCODE_SOURCE_HARVEST_ERROR => 'One or more source documents returned an error when trying to retrieve or translate',
+        ERRCODE_DOCUMENT_INVALID     => 'One or more documents to be harvested was invalid (malformed)',
+        ERRCODE_INGEST_ERROR         => 'Solr returned an error trying to ingest one or more harvested documents',
+        ERRCODE_OTHER                => 'General error code for non-harvest related issues'
       }.freeze
 
       PING_ERRCODE_MAP = {
-        'ping_solr' => ERRCODE_SOLR_PING,
-        'ping_source' => ERRCODE_SOURCE_PING,
-      }
+        'ping_solr'   => ERRCODE_SOLR_PING,
+        'ping_source' => ERRCODE_SOURCE_PING
+      }.freeze
 
       STATUS_ERRCODE_MAP = {
-          Helpers::HarvestStatus::HARVEST_NO_DOCS => ERRCODE_SOURCE_NO_RESULTS,
-          Helpers::HarvestStatus::HARVEST_FAILURE => ERRCODE_SOURCE_HARVEST_ERROR,
-          Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC => ERRCODE_DOCUMENT_INVALID,
-          Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR => ERRCODE_INGEST_ERROR,
-          Helpers::HarvestStatus::OTHER_ERROR => ERRCODE_OTHER
+        Helpers::HarvestStatus::HARVEST_NO_DOCS        => ERRCODE_SOURCE_NO_RESULTS,
+        Helpers::HarvestStatus::HARVEST_FAILURE        => ERRCODE_SOURCE_HARVEST_ERROR,
+        Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC => ERRCODE_DOCUMENT_INVALID,
+        Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR  => ERRCODE_INGEST_ERROR,
+        Helpers::HarvestStatus::OTHER_ERROR            => ERRCODE_OTHER
       }.freeze
 
       # If code is -1, it means display all error codes
       def self.describe_exit_code(code = -1)
+        code_list = code_to_list(code)
+
+        codes = {}
+        code_list.each do |k|
+          next if code == -1 && !ERRCODE_DESC.keys.include?(k) # skip INVALID CODE if showing all codes
+
+          codes[k] = ERRCODE_DESC.keys.include?(k) ? ERRCODE_DESC[k] : 'INVALID CODE NUMBER'
+        end
+
+        codes
+      end
+
+      # Loop through all bit-flag values to produce a list of integers
+      def self.code_to_list(code)
         code = code.to_i
         code_list = []
 
-        # Loop through all bit-flag values
         [128, 64, 32, 16, 8, 4, 2, 1].each do |k|
           if code >= k || code == -1
             code_list.prepend k
@@ -45,20 +60,17 @@ module SearchSolrTools
           end
         end
 
-        codes = {}
-        code_list.each do |k|
-          next if code == -1 && !ERRCODE_DESC.keys.include?(k)  # skip INVALID CODE if showing all codes
-          codes[k] = ERRCODE_DESC.keys.include?(k) ? ERRCODE_DESC[k] : 'INVALID CODE NUMBER'
-        end
-
-        codes
+        code_list
       end
 
-      def initialize(status, message=nil)
+      def initialize(status, message = nil)
         @status_data = status
         @other_message = message
+
+        super message
       end
 
+      # rubocop:disable Metrics/AbcSize
       def exit_code
         if @status_data.nil?
           puts "OTHER ERROR REPORTED: #{@other_message}"
@@ -70,19 +82,20 @@ module SearchSolrTools
         code = 0
         code += ERRCODE_SOLR_PING unless @status_data.ping_solr
         code += ERRCODE_SOURCE_PING unless @status_data.ping_source
-        code += ERRCODE_SOURCE_NO_RESULTS if @status_data.status[Helpers::HarvestStatus::HARVEST_NO_DOCS] > 0
-        code += ERRCODE_SOURCE_HARVEST_ERROR if @status_data.status[Helpers::HarvestStatus::HARVEST_FAILURE] > 0
-        code += ERRCODE_DOCUMENT_INVALID if @status_data.status[Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC] > 0
-        code += ERRCODE_INGEST_ERROR if @status_data.status[Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR] > 0
+        code += ERRCODE_SOURCE_NO_RESULTS if @status_data.status[Helpers::HarvestStatus::HARVEST_NO_DOCS].positive?
+        code += ERRCODE_SOURCE_HARVEST_ERROR if @status_data.status[Helpers::HarvestStatus::HARVEST_FAILURE].positive?
+        code += ERRCODE_DOCUMENT_INVALID if @status_data.status[Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC].positive?
+        code += ERRCODE_INGEST_ERROR if @status_data.status[Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR].positive?
 
-        code = ERRCODE_OTHER if code == 0
+        code = ERRCODE_OTHER if code.zero?
 
         code
       end
+      # rubocop:enable Metrics/AbcSize
 
       def message
-        self.class.describe_exit_code(exit_code).map{|c,v| v}.join("\n")
+        self.class.describe_exit_code(exit_code).map { |_c, v| v }.join("\n")
       end
     end
   end
-end
+end

data/lib/search_solr_tools/harvesters/auto_suggest.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'base'
 require 'json'
 require 'rest-client'
@@ -6,8 +8,8 @@ module SearchSolrTools
   module Harvesters
     # Use the nsidc_oai core to populate the auto_suggest core
     class AutoSuggest < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
+      def initialize(env = 'development', die_on_failure: false)
+        super
         @env_settings = SolrEnvironments[@environment] # super sets @environment.
       end
 
@@ -50,7 +52,7 @@ module SearchSolrTools
 
         if status == Helpers::HarvestStatus::INGEST_OK
           puts "Added #{add_docs.size} auto suggest documents in one commit"
-          return Helpers::HarvestStatus.new(Helpers::HarvestStatus::INGEST_OK => add_docs)
+          Helpers::HarvestStatus.new(Helpers::HarvestStatus::INGEST_OK => add_docs)
         else
           puts "Failed adding #{add_docs.size} documents in single commit, retrying one by one"
           new_add_docs = []

data/lib/search_solr_tools/harvesters/base.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'multi_json'
 require 'nokogiri'
 require 'open-uri'
@@ -9,7 +11,6 @@ require 'search_solr_tools'
 require_relative '../helpers/iso_namespaces'
 require_relative '../helpers/solr_format'
 
-
 module SearchSolrTools
   module Harvesters
     # base class for solr harvesters
@@ -20,7 +21,7 @@ module SearchSolrTools
       XML_CONTENT_TYPE = 'text/xml; charset=utf-8'
       JSON_CONTENT_TYPE = 'application/json; charset=utf-8'
 
-      def initialize(env = 'development', die_on_failure = false)
+      def initialize(env = 'development', die_on_failure: false)
         @environment = env
         @die_on_failure = die_on_failure
       end
@@ -51,7 +52,7 @@ module SearchSolrTools
             success = response.code == 200
             puts "Error in ping request: #{response.body}" unless success
           end
-        rescue => e
+        rescue StandardError => e
           puts "Rest exception while pinging Solr: #{e}"
         end
         success
@@ -61,7 +62,7 @@ module SearchSolrTools
       # to "ping" the data center.  Returns true if the ping is successful (or, as
       # in this default, no ping method was defined)
       def ping_source
-        puts "Harvester does not have ping method defined, assuming true"
+        puts 'Harvester does not have ping method defined, assuming true'
         true
       end
 
@@ -74,12 +75,12 @@ module SearchSolrTools
         harvest_status
       end
 
-      def delete_old_documents(timestamp, constraints, solr_core, force = false)
+      def delete_old_documents(timestamp, constraints, solr_core, force: false)
         constraints = sanitize_data_centers_constraints(constraints)
         delete_query = "last_update:[* TO #{timestamp}] AND #{constraints}"
         solr = RSolr.connect url: solr_url + "/#{solr_core}"
         unchanged_count = (solr.get 'select', params: { wt: :ruby, q: delete_query, rows: 0 })['response']['numFound'].to_i
-        if unchanged_count == 0
+        if unchanged_count.zero?
           puts "All documents were updated after #{timestamp}, nothing to delete"
         else
           puts "Begin removing documents older than #{timestamp}"
@@ -89,9 +90,9 @@ module SearchSolrTools
 
       def sanitize_data_centers_constraints(query_string)
         # Remove lucene special characters, preserve the query parameter and compress whitespace
-        query_string.gsub!(/[:&|!~\-\(\)\{\}\[\]\^\*\?\+]+/, ' ')
-        query_string.gsub!(/data_centers /, 'data_centers:')
-        query_string.gsub!(/source /, 'source:')
+        query_string = query_string.gsub(/[:&|!~\-\(\)\{\}\[\]\^\*\?\+]+/, ' ')
+        query_string = query_string.gsub('data_centers ', 'data_centers:')
+        query_string = query_string.gsub('source ', 'source:')
         query_string.squeeze(' ').strip
       end
 
@@ -126,7 +127,7 @@ module SearchSolrTools
         status
       end
 
-      # TODO Need to return a specific type of failure:
+      # TODO: Need to return a specific type of failure:
       #   - Bad record content identified and no ingest attempted
       #   - Solr tries to ingest document and fails (bad content not detected prior to ingest)
       #   - Solr cannot insert document for reasons other than the document structure and content.
@@ -142,15 +143,15 @@ module SearchSolrTools
 
         # Some docs will cause solr to time out during the POST
         begin
-          RestClient.post(url, doc_serialized, content_type: content_type) do |response, _request, _result|
+          RestClient.post(url, doc_serialized, content_type:) do |response, _request, _result|
             success = response.code == 200
             unless success
               puts "Error for #{doc_serialized}\n\n response: #{response.body}"
               status = Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR
             end
           end
-        rescue => e
-          # TODO Need to provide more detail re: this failure so we know whether to
+        rescue StandardError => e
+          # TODO: Need to provide more detail re: this failure so we know whether to
           #  exit the job with a status != 0
           puts "Rest exception while POSTing to Solr: #{e}, for doc: #{doc_serialized}"
           status = Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR
@@ -160,11 +161,11 @@ module SearchSolrTools
 
       def get_serialized_doc(doc, content_type)
         if content_type.eql?(XML_CONTENT_TYPE)
-          return doc.respond_to?(:to_xml) ? doc.to_xml : doc
+          doc.respond_to?(:to_xml) ? doc.to_xml : doc
         elsif content_type.eql?(JSON_CONTENT_TYPE)
-          return MultiJson.dump(doc)
+          MultiJson.dump(doc)
         else
-          return doc
+          doc
         end
       end
 
@@ -177,17 +178,18 @@ module SearchSolrTools
 
         begin
           puts "Request: #{request_url}"
-          response = URI.open(request_url, read_timeout: timeout, 'Content-Type' => content_type)
+          response = URI.parse(request_url).open(read_timeout: timeout, 'Content-Type' => content_type)
         rescue OpenURI::HTTPError, Timeout::Error, Errno::ETIMEDOUT => e
           retries_left -= 1
           puts "## REQUEST FAILED ## #{e.class} ## Retrying #{retries_left} more times..."
 
-          retry if retries_left > 0
+          retry if retries_left.positive?
 
-          # TODO - Do we really need this "die_on_failure" anymore?  The empty return
+          # TODO: Do we really need this "die_on_failure" anymore?  The empty return
           #  will cause the "No Documents" error to be thrown in the harvester class
           #  now, so it will pretty much always "die on failure"
           raise e if @die_on_failure
+
           return
         end
         doc = Nokogiri.XML(response)
@@ -215,7 +217,7 @@ module SearchSolrTools
         spatial_coverages = doc.xpath(".//field[@name='spatial_coverages']").first
         return true if spatial_coverages.nil?
 
-        spatial_coverages = spatial_coverages.text.split(' ')
+        spatial_coverages = spatial_coverages.text.split
 
         # We've only seen the failure with 4 spatial coverage values
         return true if spatial_coverages.size < 4

data/lib/search_solr_tools/harvesters/nsidc_auto_suggest.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'auto_suggest'
 
 module SearchSolrTools
@@ -16,11 +18,11 @@ module SearchSolrTools
       def fields
         {
           'authoritative_id' => { weight: 1, source: 'NSIDC', creator: method(:standard_add_creator) },
-          'full_title' => { weight: 2, source: 'NSIDC', creator: method(:standard_add_creator) },
-          'copy_parameters' => { weight: 5, source: 'NSIDC', creator: method(:standard_add_creator) },
-          'full_platforms' => { weight: 2, source: 'NSIDC', creator: method(:short_full_split_add_creator) },
-          'full_sensors' => { weight: 2, source: 'NSIDC', creator: method(:short_full_split_add_creator) },
-          'full_authors' => { weight: 1, source: 'NSIDC', creator: method(:standard_add_creator) }
+          'full_title'       => { weight: 2, source: 'NSIDC', creator: method(:standard_add_creator) },
+          'copy_parameters'  => { weight: 5, source: 'NSIDC', creator: method(:standard_add_creator) },
+          'full_platforms'   => { weight: 2, source: 'NSIDC', creator: method(:short_full_split_add_creator) },
+          'full_sensors'     => { weight: 2, source: 'NSIDC', creator: method(:short_full_split_add_creator) },
+          'full_authors'     => { weight: 1, source: 'NSIDC', creator: method(:standard_add_creator) }
         }
       end
 

data/lib/search_solr_tools/harvesters/nsidc_json.rb

@@ -1,15 +1,16 @@
+# frozen_string_literal: true
+
 require 'json'
 require 'rest-client'
 
 require 'search_solr_tools'
 
-
 module SearchSolrTools
   module Harvesters
     # Harvests data from NSIDC OAI and inserts it into Solr after it has been translated
     class NsidcJson < Base
-      def initialize(env = 'development', die_on_failure = false)
-        super env, die_on_failure
+      def initialize(env = 'development', die_on_failure: false)
+        super
         @translator = Translators::NsidcJsonToSolr.new
         Helpers::FacetConfiguration.import_bin_configuration(env)
       end
@@ -19,7 +20,7 @@ module SearchSolrTools
           RestClient.options(nsidc_json_url) do |response, _request, _result|
             return response.code == 200
           end
-        rescue => e
+        rescue StandardError
           puts "Error trying to get options for #{nsidc_json_url} (ping)"
         end
         false
@@ -37,7 +38,7 @@ module SearchSolrTools
 
         status = insert_solr_docs result[:add_docs], Base::JSON_CONTENT_TYPE
 
-        status.record_status(Helpers::HarvestStatus::HARVEST_NO_DOCS) if result[:num_docs] == 0
+        status.record_status(Helpers::HarvestStatus::HARVEST_NO_DOCS) if (result[:num_docs]).zero?
 
         # Record the number of harvest failures; note that if this is 0, thats OK, the status will stay at 0
         status.record_status(Helpers::HarvestStatus::HARVEST_FAILURE, result[:failure_ids].length)
@@ -66,7 +67,7 @@ module SearchSolrTools
       # @param id [String] NSIDC authoritative ID for the dataset
       # @return [Hash] Parsed version of the JSON response
       def fetch_json_from_nsidc(id)
-        json_response = RestClient.get(nsidc_json_url + id + '.json')
+        json_response = RestClient.get("#{nsidc_json_url}#{id}.json")
         JSON.parse(json_response)
       end
 
@@ -81,13 +82,13 @@ module SearchSolrTools
           id = r.text.split('/').last
           begin
             docs << { 'add' => { 'doc' => @translator.translate(fetch_json_from_nsidc(id)) } }
-          rescue => e
+          rescue StandardError => e
             puts "Failed to fetch #{id} with error #{e}: #{e.backtrace}"
             failure_ids << id
           end
         end
 
-        { num_docs: all_docs.size, add_docs: docs, failure_ids: failure_ids }
+        { num_docs: all_docs.size, add_docs: docs, failure_ids: }
       end
     end
   end

data/lib/search_solr_tools/helpers/bounding_box_util.rb

@@ -1,4 +1,6 @@
-require_relative './iso_namespaces'
+# frozen_string_literal: true
+
+require_relative 'iso_namespaces'
 
 module SearchSolrTools
   module Helpers
@@ -8,12 +10,10 @@ module SearchSolrTools
       NORTHERN_GLOBAL_BOUNDARY = 85.0
 
       def self.bounding_box_hash_from_geo_json(geometry)
-        if geometry_is_point?(geometry)
-          return { west: geometry.x.to_s, south: geometry.y.to_s, east: geometry.x.to_s, north: geometry.y.to_s }
-        else
-          bbox = RGeo::Cartesian::BoundingBox.create_from_geometry(geometry)
-          return { west: bbox.min_x.to_s, south: bbox.min_y.to_s, east: bbox.max_x.to_s, north: bbox.max_y.to_s }
-        end
+        return { west: geometry.x.to_s, south: geometry.y.to_s, east: geometry.x.to_s, north: geometry.y.to_s } if geometry_is_point?(geometry)
+
+        bbox = RGeo::Cartesian::BoundingBox.create_from_geometry(geometry)
+        { west: bbox.min_x.to_s, south: bbox.min_y.to_s, east: bbox.max_x.to_s, north: bbox.max_y.to_s }
       end
 
       def self.geometry_is_point?(geometry)
@@ -30,7 +30,7 @@ module SearchSolrTools
       end
 
       def self.box_invalid?(box)
-        [:north, :south, :east, :west].any? { |d| box[d].to_s.empty? }
+        %i[north south east west].any? { |d| box[d].to_s.empty? }
       end
     end
   end

data/lib/search_solr_tools/helpers/facet_configuration.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'json'
 require 'rest_client'
 require 'singleton'
@@ -8,7 +10,7 @@ module SearchSolrTools
     class FacetConfiguration
       include Singleton
       def self.import_bin_configuration(env)
-        @bin_configuration = JSON.parse(RestClient.get(SolrEnvironments[env][:nsidc_dataset_metadata_url] + 'binConfiguration')) if @bin_configuration.nil?
+        @bin_configuration = JSON.parse(RestClient.get("#{SolrEnvironments[env][:nsidc_dataset_metadata_url]}binConfiguration")) if @bin_configuration.nil?
       end
 
       def self.get_facet_bin(facet_name)

data/lib/search_solr_tools/helpers/harvest_status.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SearchSolrTools
   module Helpers
     class HarvestStatus
@@ -7,17 +9,17 @@ module SearchSolrTools
       INGEST_ERR_INVALID_DOC = :invalid
       INGEST_ERR_SOLR_ERROR = :solr_error
       OTHER_ERROR = :other
-      PING_SOLR = :ping_solr  # used for initialize only
-      PING_SOURCE = :ping_source  # used for initialize only
+      PING_SOLR = :ping_solr # used for initialize only
+      PING_SOURCE = :ping_source # used for initialize only
 
-      ERROR_STATUS = [HARVEST_NO_DOCS, HARVEST_FAILURE, INGEST_ERR_INVALID_DOC, INGEST_ERR_SOLR_ERROR, OTHER_ERROR]
+      ERROR_STATUS = [HARVEST_NO_DOCS, HARVEST_FAILURE, INGEST_ERR_INVALID_DOC, INGEST_ERR_SOLR_ERROR, OTHER_ERROR].freeze
 
-      attr_reader :status, :ping_solr, :ping_source
-      attr_writer :ping_solr, :ping_source
+      attr_accessor :ping_solr, :ping_source
+      attr_reader :status
 
       # init_info is an optional hash that contains the various status keys and the documents to
       # associate with them
-      def initialize(init_info={})
+      def initialize(init_info = {})
         @status = { INGEST_OK => 0 }
         @ping_solr = true
         @ping_source = true
@@ -36,9 +38,9 @@ module SearchSolrTools
       end
 
       def ok?
-        ERROR_STATUS.each { |s| return false unless @status[s] == 0 }
+        ERROR_STATUS.each { |s| return false unless (@status[s]).zero? }
         @ping_solr && @ping_source
       end
     end
   end
-end
+end

data/lib/search_solr_tools/helpers/iso_namespaces.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SearchSolrTools
   module Helpers
     # Helper class to provide default namespaces for XML document parsing.
@@ -25,7 +27,7 @@ module SearchSolrTools
         'srv'    => 'http://www.isotc211.org/2005/srv',
         'xlink'  => 'http://www.w3.org/1999/xlink',
         'xsi'    => 'http://www.w3.org/2001/XMLSchema-instance'
-      }
+      }.freeze
     end
   end
 end

data/lib/search_solr_tools/helpers/solr_format.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'date'
 require 'iso8601'
 
@@ -7,15 +9,14 @@ require_relative 'facet_configuration'
 module SearchSolrTools
   module Helpers
     #  Methods for generating formatted values that can be indexed by SOLR
-    # rubocop:disable Metrics/ModuleLength
     module SolrFormat
       DATA_CENTER_NAMES = {
-        NSIDC: { short_name: 'NSIDC', long_name: 'National Snow and Ice Data Center' },
-      }
+        NSIDC: { short_name: 'NSIDC', long_name: 'National Snow and Ice Data Center' }
+      }.freeze
 
       NOT_SPECIFIED = 'Not specified'
 
-      TEMPORAL_RESOLUTION_FACET_VALUES = %w(Subhourly Hourly Subdaily Daily Weekly Submonthly Monthly Subyearly Yearly Multiyearly)
+      TEMPORAL_RESOLUTION_FACET_VALUES = %w[Subhourly Hourly Subdaily Daily Weekly Submonthly Monthly Subyearly Yearly Multiyearly].freeze
       SUBHOURLY_INDEX = 0
       HOURLY_INDEX = 1
       SUBDAILY_INDEX = 2
@@ -27,7 +28,7 @@ module SearchSolrTools
       YEARLY_INDEX = 8
       MULTIYEARLY_INDEX = 9
 
-      SPATIAL_RESOLUTION_FACET_VALUES = ['0 - 500 m', '501 m - 1 km', '2 - 5 km', '6 - 15 km', '16 - 30 km', '>30 km']
+      SPATIAL_RESOLUTION_FACET_VALUES = ['0 - 500 m', '501 m - 1 km', '2 - 5 km', '6 - 15 km', '16 - 30 km', '>30 km'].freeze
       SPATIAL_0_500_INDEX = 0
       SPATIAL_501_1_INDEX = 1
       SPATIAL_2_5_INDEX = 2
@@ -44,7 +45,7 @@ module SearchSolrTools
       end
 
       def self.temporal_display_str(date_range)
-        temporal_str = "#{date_range[:start]}"
+        temporal_str = (date_range[:start]).to_s
         temporal_str += ",#{date_range[:end]}" unless date_range[:end].nil?
         temporal_str
       end
@@ -67,6 +68,7 @@ module SearchSolrTools
 
       def self.get_temporal_duration_facet(duration)
         return NOT_SPECIFIED if duration.nil?
+
         years = duration.to_i / 365
         temporal_duration_range(years)
       end
@@ -86,31 +88,28 @@ module SearchSolrTools
       def self.facet_binning(type, format_string)
         binned_facet = bin(FacetConfiguration.get_facet_bin(type), format_string)
         if binned_facet.nil?
-          return format_string
+          format_string
         elsif binned_facet.eql?('exclude')
-          return nil
+          nil
         else
-          return binned_facet
+          binned_facet
         end
-
-        nil
       end
 
       def self.parameter_binning(parameter_string)
         binned_parameter = bin(FacetConfiguration.get_facet_bin('parameter'), parameter_string)
         # use variable_level_1 if no mapping exists
-        if binned_parameter.nil?
-          parts = parameter_string.split '>'
-          return parts[3].strip if parts.length >= 4
-        else
-          return binned_parameter
-        end
+        return binned_parameter unless binned_parameter.nil?
+
+        parts = parameter_string.split '>'
+        return parts[3].strip if parts.length >= 4
 
         nil
       end
 
       def self.resolution_value(resolution, find_index_method, resolution_values)
-        return NOT_SPECIFIED if self.resolution_not_specified? resolution
+        return NOT_SPECIFIED if resolution_not_specified? resolution
+
         if resolution['type'] == 'single'
           i = send(find_index_method, resolution['resolution'])
           return resolution_values[i]
@@ -120,12 +119,12 @@ module SearchSolrTools
           j = send(find_index_method, resolution['max_resolution'])
           return resolution_values[i..j]
         end
-        fail "Invalid resolution #{resolution['type']}"
+        raise "Invalid resolution #{resolution['type']}"
       end
 
       def self.resolution_not_specified?(resolution)
         return true if resolution.to_s.empty?
-        return true unless %w(single range).include? resolution['type']
+        return true unless %w[single range].include? resolution['type']
         return true if resolution['type'] == 'single' && resolution['resolution'].to_s.empty?
         return true if resolution['type'] == 'range' && resolution['min_resolution'].to_s.empty?
       end
@@ -140,6 +139,7 @@ module SearchSolrTools
         else
           facet = 'Between 1 and 170 degrees of latitude change | Regional'
         end
+
         facet
       end
 
@@ -152,8 +152,6 @@ module SearchSolrTools
         "#{d.iso8601[0..-7]}Z" unless d.nil?
       end
 
-      private
-
       MIN_DATE = '00010101'
       MAX_DATE = Time.now.strftime('%Y%m%d')
 
@@ -166,7 +164,6 @@ module SearchSolrTools
         nil
       end
 
-      # rubocop:disable CyclomaticComplexity
       def self.find_index_for_single_temporal_resolution_value(string_duration)
         iso8601_duration = ISO8601::Duration.new(string_duration)
 
@@ -186,10 +183,9 @@ module SearchSolrTools
           MULTIYEARLY_INDEX
         end
       end
-      # rubocop:enable CyclomaticComplexity
 
       def self.find_index_for_single_spatial_resolution_value(string_duration)
-        value, units = string_duration.split(' ')
+        value, units = string_duration.split
 
         if units == 'deg'
           spatial_resolution_index_degrees(value)
@@ -234,11 +230,10 @@ module SearchSolrTools
       end
 
       def self.date?(date)
-        valid_date = if date.is_a? String
-                       d = DateTime.parse(date.strip) rescue false
-                       DateTime.valid_date?(d.year, d.mon, d.day) unless d.eql?(false)
-                     end
-        valid_date
+        return false unless date.is_a? String
+
+        d = DateTime.parse(date.strip) rescue false
+        DateTime.valid_date?(d.year, d.mon, d.day) unless d.eql?(false)
       end
 
       def self.format_date_for_index(date_str, default)

data/lib/search_solr_tools/helpers/translate_spatial_coverage.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rgeo/geo_json'
 
 require_relative 'bounding_box_util'
@@ -42,7 +44,7 @@ module SearchSolrTools
 
       def self.geojson_to_spatial_area(spatial_coverage_geom)
         spatial_areas = spatial_coverage_geom.map do |geo_json|
-          if %w(point).include?(geo_json.geometry_type.to_s.downcase)
+          if %w[point].include?(geo_json.geometry_type.to_s.downcase)
             0.0
           else
             bbox = RGeo::Cartesian::BoundingBox.create_from_geometry(geo_json)
@@ -50,11 +52,13 @@ module SearchSolrTools
           end
         end
         return nil if spatial_areas.empty?
-        spatial_areas.sort.last
+
+        spatial_areas.max
       end
 
       def self.geojson_to_global_facet(spatial_coverage_geom)
         return nil if spatial_coverage_geom.nil?
+
         spatial_coverage_geom.each do |geo_json|
           bbox_hash = BoundingBoxUtil.bounding_box_hash_from_geo_json(geo_json)
           return 'Show Global Only' if BoundingBoxUtil.box_global?(bbox_hash)
@@ -63,13 +67,13 @@ module SearchSolrTools
       end
 
       def self.geojson_to_spatial_scope_facet(spatial_coverage_geom)
-        unless spatial_coverage_geom.nil?
-          spatial_coverage_geom.map do |geo_json|
-            bbox_hash = BoundingBoxUtil.bounding_box_hash_from_geo_json(geo_json)
-            scope = SolrFormat.get_spatial_scope_facet_with_bounding_box(bbox_hash)
-            scope unless scope.nil?
-          end.uniq
-        end
+        return if spatial_coverage_geom.nil?
+
+        spatial_coverage_geom.map do |geo_json|
+          bbox_hash = BoundingBoxUtil.bounding_box_hash_from_geo_json(geo_json)
+          scope = SolrFormat.get_spatial_scope_facet_with_bounding_box(bbox_hash)
+          scope unless scope.nil?
+        end.uniq
       end
     end
   end

data/lib/search_solr_tools/helpers/translate_temporal_coverage.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rgeo/geo_json'
 
 require_relative 'solr_format'

data/lib/search_solr_tools/translators/nsidc_json.rb

@@ -1,4 +1,5 @@
-# rubocop:disable Metrics/ClassLength
+# frozen_string_literal: true
+
 require 'rgeo/geo_json'
 
 require 'search_solr_tools'
@@ -10,50 +11,50 @@ module SearchSolrTools
   module Translators
     # Translates NSIDC JSON format to Solr JSON add format
     class NsidcJsonToSolr
-      PARAMETER_PARTS = %w(category topic term variableLevel1 variableLevel2 variableLevel3 detailedVariable)
+      PARAMETER_PARTS = %w[category topic term variableLevel1 variableLevel2 variableLevel3 detailedVariable].freeze
 
       # rubocop:disable Metrics/MethodLength
       # rubocop:disable Metrics/AbcSize
       def translate(json_doc)
-        copy_keys = %w(title summary keywords brokered)
+        copy_keys = %w[title summary keywords brokered]
         temporal_coverage_values = Helpers::TranslateTemporalCoverage.translate_coverages json_doc['temporalCoverages']
         spatial_coverages = convert_spatial_coverages(json_doc['spatialCoverages'])
 
         solr_add_hash = json_doc.select { |k, _v| copy_keys.include?(k) }
         solr_add_hash.merge!(
-          'authoritative_id' => json_doc['authoritativeId'],
-          'dataset_version' => json_doc['majorVersion']['version'],
-          'data_centers' => Helpers::SolrFormat::DATA_CENTER_NAMES[:NSIDC][:long_name],
-          'facet_data_center' => "#{Helpers::SolrFormat::DATA_CENTER_NAMES[:NSIDC][:long_name]} | #{Helpers::SolrFormat::DATA_CENTER_NAMES[:NSIDC][:short_name]}",
-          'authors' => translate_personnel_and_creators_to_authors(json_doc['personnel'], generate_data_citation_creators(json_doc['dataCitation'])),
-          'topics' => translate_iso_topic_categories(json_doc['isoTopicCategories']),
-          'parameters' => translate_parameters(json_doc['parameters']),
-          'full_parameters' => translate_json_string(json_doc['parameters'], PARAMETER_PARTS),
-          'facet_parameter' => translate_parameters_to_facet_parameters(json_doc['parameters']),
-          'platforms' => translate_json_string(json_doc['platforms']),
-          'sensors' => translate_json_string(json_doc['instruments']),
-          'facet_sensor' => translate_sensor_to_facet_sensor(json_doc['instruments']),
-          'published_date' => (Helpers::SolrFormat.date_str json_doc['releaseDate']),
-          'spatial_coverages' => Helpers::TranslateSpatialCoverage.geojson_to_spatial_display_str(spatial_coverages),
-          'spatial' => Helpers::TranslateSpatialCoverage.geojson_to_spatial_index_str(spatial_coverages),
-          'spatial_area' => Helpers::TranslateSpatialCoverage.geojson_to_spatial_area(spatial_coverages),
-          'facet_spatial_coverage' => Helpers::TranslateSpatialCoverage.geojson_to_global_facet(spatial_coverages),
-          'facet_spatial_scope' => Helpers::TranslateSpatialCoverage.geojson_to_spatial_scope_facet(spatial_coverages),
-          'temporal_coverages' => temporal_coverage_values['temporal_coverages'],
-          'temporal_duration' => temporal_coverage_values['temporal_duration'],
-          'temporal' => temporal_coverage_values['temporal'],
-          'facet_temporal_duration' => temporal_coverage_values['facet_temporal_duration'],
-          'last_revision_date' => (Helpers::SolrFormat.date_str json_doc['lastRevisionDate']),
-          'dataset_url' => json_doc['datasetUrl'],
-          'distribution_formats' => json_doc['distributionFormats'],
-          'facet_format' => json_doc['distributionFormats'].empty? ? [Helpers::SolrFormat::NOT_SPECIFIED] : translate_format_to_facet_format(json_doc['distributionFormats']),
-          'source' => %w(NSIDC ADE),
-          'popularity' => json_doc['popularity'],
-          'data_access_urls' => translate_data_access_urls(json_doc['dataAccessLinks']),
-          'facet_sponsored_program' => translate_short_long_names_to_facet_value(json_doc['internalDataCenters']),
+          'authoritative_id'          => json_doc['authoritativeId'],
+          'dataset_version'           => json_doc['majorVersion']['version'],
+          'data_centers'              => Helpers::SolrFormat::DATA_CENTER_NAMES[:NSIDC][:long_name],
+          'facet_data_center'         => "#{Helpers::SolrFormat::DATA_CENTER_NAMES[:NSIDC][:long_name]} | #{Helpers::SolrFormat::DATA_CENTER_NAMES[:NSIDC][:short_name]}",
+          'authors'                   => translate_personnel_and_creators_to_authors(json_doc['personnel'], generate_data_citation_creators(json_doc['dataCitation'])),
+          'topics'                    => translate_iso_topic_categories(json_doc['isoTopicCategories']),
+          'parameters'                => translate_parameters(json_doc['parameters']),
+          'full_parameters'           => translate_json_string(json_doc['parameters'], PARAMETER_PARTS),
+          'facet_parameter'           => translate_parameters_to_facet_parameters(json_doc['parameters']),
+          'platforms'                 => translate_json_string(json_doc['platforms']),
+          'sensors'                   => translate_json_string(json_doc['instruments']),
+          'facet_sensor'              => translate_sensor_to_facet_sensor(json_doc['instruments']),
+          'published_date'            => (Helpers::SolrFormat.date_str json_doc['releaseDate']),
+          'spatial_coverages'         => Helpers::TranslateSpatialCoverage.geojson_to_spatial_display_str(spatial_coverages),
+          'spatial'                   => Helpers::TranslateSpatialCoverage.geojson_to_spatial_index_str(spatial_coverages),
+          'spatial_area'              => Helpers::TranslateSpatialCoverage.geojson_to_spatial_area(spatial_coverages),
+          'facet_spatial_coverage'    => Helpers::TranslateSpatialCoverage.geojson_to_global_facet(spatial_coverages),
+          'facet_spatial_scope'       => Helpers::TranslateSpatialCoverage.geojson_to_spatial_scope_facet(spatial_coverages),
+          'temporal_coverages'        => temporal_coverage_values['temporal_coverages'],
+          'temporal_duration'         => temporal_coverage_values['temporal_duration'],
+          'temporal'                  => temporal_coverage_values['temporal'],
+          'facet_temporal_duration'   => temporal_coverage_values['facet_temporal_duration'],
+          'last_revision_date'        => (Helpers::SolrFormat.date_str json_doc['lastRevisionDate']),
+          'dataset_url'               => json_doc['datasetUrl'],
+          'distribution_formats'      => json_doc['distributionFormats'],
+          'facet_format'              => json_doc['distributionFormats'].empty? ? [Helpers::SolrFormat::NOT_SPECIFIED] : translate_format_to_facet_format(json_doc['distributionFormats']),
+          'source'                    => %w[NSIDC ADE],
+          'popularity'                => json_doc['popularity'],
+          'data_access_urls'          => translate_data_access_urls(json_doc['dataAccessLinks']),
+          'facet_sponsored_program'   => translate_short_long_names_to_facet_value(json_doc['internalDataCenters']),
           'facet_temporal_resolution' => translate_temporal_resolution_facet_values(json_doc['parameters']),
-          'facet_spatial_resolution' => translate_spatial_resolution_facet_values(json_doc['parameters']),
-          'sponsored_programs' => translate_internal_datacenters(json_doc['internalDataCenters'])
+          'facet_spatial_resolution'  => translate_spatial_resolution_facet_values(json_doc['parameters']),
+          'sponsored_programs'        => translate_internal_datacenters(json_doc['internalDataCenters'])
         )
       end
       # rubocop:enable Metrics/MethodLength
@@ -70,13 +71,14 @@ module SearchSolrTools
       def translate_sensor_to_facet_sensor(json)
         facet_values = []
         return facet_values if json.nil?
+
         json.each do |json_entry|
           sensor_bin = Helpers::SolrFormat.facet_binning('sensor', json_entry['shortName'].to_s)
-          if sensor_bin.eql? json_entry['shortName']
-            facet_values << "#{json_entry['longName']} | #{json_entry['shortName']}"
-          else
-            facet_values << " | #{sensor_bin}"
-          end
+          facet_values << if sensor_bin.eql? json_entry['shortName']
+                            "#{json_entry['longName']} | #{json_entry['shortName']}"
+                          else
+                            " | #{sensor_bin}"
+                          end
         end
         facet_values
       end
@@ -100,12 +102,13 @@ module SearchSolrTools
       end
 
       def translate_iso_topic_categories(iso_topic_categories_json)
-        iso_topic_categories_json.map { |t| t['name'] } unless iso_topic_categories_json.nil?
+        iso_topic_categories_json&.map { |t| t['name'] }
       end
 
       def translate_data_access_urls(json)
         values = []
         return values if json.nil?
+
         json.each do |json_entry|
           link_display = json_entry['displayText'].nil? ? '' : json_entry['displayText']
           link_type = json_entry['type'].nil? ? '' : json_entry['type']
@@ -120,6 +123,7 @@ module SearchSolrTools
       def translate_internal_datacenters(json)
         values = []
         return values if json.nil?
+
         json.each do |json_entry|
           short_name = json_entry['shortName'].nil? ? '' : json_entry['shortName']
           values << short_name
@@ -130,6 +134,7 @@ module SearchSolrTools
       def translate_short_long_names_to_facet_value(json)
         facet_values = []
         return facet_values if json.nil?
+
         json.each do |json_entry|
           long_name = json_entry['longName'].nil? ? '' : json_entry['longName']
           short_name = json_entry['shortName'].nil? ? '' : json_entry['shortName']
@@ -169,6 +174,7 @@ module SearchSolrTools
       def translate_parameters_to_facet_parameters(parameters_json)
         parameters_strings = translate_json_string(parameters_json, PARAMETER_PARTS)
         return [] if parameters_strings.nil?
+
         facet_params = []
         parameters_strings.each do |str|
           facet_params << Helpers::SolrFormat.parameter_binning(str)
@@ -199,8 +205,7 @@ module SearchSolrTools
       end
 
       def generate_data_citation_creators(data_citation)
-        data_citation.nil? ? creators = [] : creators = data_citation['creators']
-        creators
+        data_citation.nil? ? [] : data_citation['creators']
       end
 
       def generate_part_array(json, limit_values = nil)
@@ -214,6 +219,5 @@ module SearchSolrTools
         parts
       end
     end
-    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/search_solr_tools/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SearchSolrTools
-  VERSION = '6.2.0'
+  VERSION = '6.4.0'
 end

data/lib/search_solr_tools.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 require_relative 'search_solr_tools/config/environments'
 require_relative 'search_solr_tools/version'
 
 require_relative 'search_solr_tools/helpers/harvest_status'
 require_relative 'search_solr_tools/errors/harvest_error'
 
-%w( harvesters translators ).each do |subdir|
+%w[harvesters translators].each do |subdir|
   Dir[File.join(__dir__, 'search_solr_tools', subdir, '*.rb')].each { |file| require file }
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: search_solr_tools
 version: !ruby/object:Gem::Version
-  version: 6.2.0
+  version: 6.4.0
 platform: ruby
 authors:
 - Chris Chalstrom
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-07-18 00:00:00.000000000 Z
+date: 2023-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi-geos
@@ -354,7 +354,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.4.17
 signing_key: 
 specification_version: 4
 summary: Tools to harvest and manage various scientific dataset feeds in a Solr instance.