search_solr_tools 4.2.0 → 5.1.0

data/lib/search_solr_tools/harvesters/bcodmo.rb

@@ -1,3 +1,4 @@
+require_relative 'base'
 require 'json'
 require 'rest-client'
 

data/lib/search_solr_tools/harvesters/data_one.rb

@@ -1,3 +1,5 @@
+require_relative 'base'
+
 module SearchSolrTools
   module Harvesters
     class DataOne < Base

data/lib/search_solr_tools/harvesters/echo.rb

@@ -1,3 +1,5 @@
+require_relative 'base'
+
 module SearchSolrTools
   module Harvesters
     # Harvests data from ECHO and inserts it into Solr after it has been translated

data/lib/search_solr_tools/harvesters/gtnp.rb

@@ -1,3 +1,4 @@
+require_relative 'base'
 require 'json'
 require 'rest-client'
 

data/lib/search_solr_tools/harvesters/ices.rb

@@ -1,3 +1,6 @@
+require_relative 'base'
+require_relative '../helpers/csw_iso_query_builder'
+
 module SearchSolrTools
   module Harvesters
     # Harvests data from ICES and inserts it into Solr after it has been translated

data/lib/search_solr_tools/harvesters/ncdc_paleo.rb

@@ -1,3 +1,6 @@
+require_relative 'base'
+require_relative '../helpers/csw_iso_query_builder'
+
 module SearchSolrTools
   module Harvesters
     # Harvests data from NODC PALEO and inserts it into Solr after it has been translated

data/lib/search_solr_tools/harvesters/nmi.rb

@@ -1,3 +1,5 @@
+require_relative 'oai'
+
 module SearchSolrTools
   module Harvesters
     class Nmi < Oai

data/lib/search_solr_tools/harvesters/nodc.rb

@@ -1,3 +1,6 @@
+require_relative 'base'
+require_relative '../helpers/csw_iso_query_builder'
+
 module SearchSolrTools
   module Harvesters
     # Harvests data from NODC and inserts it into Solr after it has been translated

data/lib/search_solr_tools/harvesters/nsidc_auto_suggest.rb

@@ -1,3 +1,5 @@
+require_relative 'auto_suggest'
+
 module SearchSolrTools
   module Harvesters
     class NsidcAutoSuggest < AutoSuggest

data/lib/search_solr_tools/harvesters/nsidc_json.rb

@@ -3,6 +3,7 @@ require 'rest-client'
 
 require 'search_solr_tools'
 
+
 module SearchSolrTools
   module Harvesters
     # Harvests data from NSIDC OAI and inserts it into Solr after it has been translated
@@ -13,6 +14,17 @@ module SearchSolrTools
         Helpers::FacetConfiguration.import_bin_configuration(env)
       end
 
+      def ping_source
+        begin
+          RestClient.options(nsidc_json_url) do |response, _request, _result|
+            return response.code == 200
+          end
+        rescue => e
+          puts "Error trying to get options for #{nsidc_json_url} (ping)"
+        end
+        false
+      end
+
       def harvest_and_delete
         puts "Running harvest of NSIDC catalog from #{nsidc_json_url}"
         super(method(:harvest_nsidc_json_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:NSIDC][:long_name]}\"")
@@ -22,8 +34,22 @@ module SearchSolrTools
       # this is the main entry point for the class
       def harvest_nsidc_json_into_solr
         result = docs_with_translated_entries_from_nsidc
-        insert_solr_docs result[:add_docs], Base::JSON_CONTENT_TYPE
-        fail 'Failed to harvest and insert some authoritative IDs' if result[:failure_ids].length > 0
+
+        status = insert_solr_docs result[:add_docs], Base::JSON_CONTENT_TYPE
+
+        status.record_status(Helpers::HarvestStatus::HARVEST_NO_DOCS) if result[:num_docs] == 0
+
+        # Record the number of harvest failures; note that if this is 0, thats OK, the status will stay at 0
+        status.record_status(Helpers::HarvestStatus::HARVEST_FAILURE, result[:failure_ids].length)
+
+        raise Errors::HarvestError, status unless status.ok?
+      rescue Errors::HarvestError => e
+        raise e
+      rescue StandardError => e
+        puts "An unexpected exception occurred while trying to harvest or insert: #{e}"
+        puts e.backtrace
+        status = Helpers::HarvestStatus.new(Helpers::HarvestStatus::OTHER_ERROR => e)
+        raise Errors::HarvestError, status
       end
 
       def nsidc_json_url
@@ -33,7 +59,7 @@ module SearchSolrTools
       def result_ids_from_nsidc
         url = SolrEnvironments[@environment][:nsidc_dataset_metadata_url] +
               SolrEnvironments[@environment][:nsidc_oai_identifiers_url]
-        get_results url, '//xmlns:identifier'
+        get_results(url, '//xmlns:identifier') || []
       end
 
       # Fetch a JSON representation of a dataset's metadata
@@ -48,7 +74,8 @@ module SearchSolrTools
         docs = []
         failure_ids = []
 
-        result_ids_from_nsidc.each do |r|
+        all_docs = result_ids_from_nsidc
+        all_docs.each do |r|
           # Each result looks like:
           # oai:nsidc.org/AE_L2A
           id = r.text.split('/').last
@@ -60,7 +87,7 @@ module SearchSolrTools
           end
         end
 
-        { add_docs: docs, failure_ids: failure_ids }
+        { num_docs: all_docs.size, add_docs: docs, failure_ids: failure_ids }
       end
     end
   end

data/lib/search_solr_tools/harvesters/oai.rb

@@ -1,3 +1,6 @@
+require_relative 'base'
+require_relative '../helpers/query_builder'
+
 require 'json'
 
 module SearchSolrTools

data/lib/search_solr_tools/harvesters/pdc.rb

@@ -1,3 +1,5 @@
+require_relative 'oai'
+
 module SearchSolrTools
   module Harvesters
     # Harvests data from Polar data catalogue and inserts it into

data/lib/search_solr_tools/harvesters/r2r.rb

@@ -1,8 +1,8 @@
+require_relative 'base'
+
 require 'nokogiri'
 require 'rest-client'
 
-require_relative 'base'
-
 module SearchSolrTools
   module Harvesters
     class R2R < Base

data/lib/search_solr_tools/harvesters/rda.rb

@@ -1,3 +1,5 @@
+require_relative 'oai'
+
 module SearchSolrTools
   module Harvesters
     # Harvests the RDA feed

data/lib/search_solr_tools/harvesters/tdar.rb

@@ -1,3 +1,5 @@
+require_relative 'base'
+
 module SearchSolrTools
   module Harvesters
     # Harvests data from TDAR and inserts it into Solr after it has been translated

data/lib/search_solr_tools/harvesters/usgs.rb

@@ -1,3 +1,6 @@
+require_relative 'base'
+require_relative '../helpers/csw_iso_query_builder'
+
 module SearchSolrTools
   module Harvesters
     # Harvests data from USGS and inserts it into Solr after it has been translated

data/lib/search_solr_tools/helpers/data_one_format.rb

@@ -1,6 +1,6 @@
-require_relative './iso_namespaces'
-require_relative './iso_to_solr_format'
-require_relative './solr_format'
+require_relative 'iso_namespaces'
+require_relative 'iso_to_solr_format'
+require_relative 'solr_format'
 
 module SearchSolrTools
   module Helpers

data/lib/search_solr_tools/helpers/harvest_status.rb

@@ -0,0 +1,44 @@
+module SearchSolrTools
+  module Helpers
+    class HarvestStatus
+      INGEST_OK = :ok
+      HARVEST_NO_DOCS = :harvest_none
+      HARVEST_FAILURE = :harvest_fail
+      INGEST_ERR_INVALID_DOC = :invalid
+      INGEST_ERR_SOLR_ERROR = :solr_error
+      OTHER_ERROR = :other
+      PING_SOLR = :ping_solr  # used for initialize only
+      PING_SOURCE = :ping_source  # used for initialize only
+
+      ERROR_STATUS = [HARVEST_NO_DOCS, HARVEST_FAILURE, INGEST_ERR_INVALID_DOC, INGEST_ERR_SOLR_ERROR, OTHER_ERROR]
+
+      attr_reader :status, :ping_solr, :ping_source
+      attr_writer :ping_solr, :ping_source
+
+      # init_info is an optional hash that contains the various status keys and the documents to
+      # associate with them
+      def initialize(init_info={})
+        @status = { INGEST_OK => 0 }
+        @ping_solr = true
+        @ping_source = true
+        ERROR_STATUS.each { |s| @status[s] = 0 }
+
+        init_info.each do |key, count|
+          @status[key] = count if @status.include? key
+        end
+
+        @ping_solr = init_info[PING_SOLR] if init_info.include? PING_SOLR
+        @ping_source = init_info[PING_SOURCE] if init_info.include? PING_SOURCE
+      end
+
+      def record_status(status, count = 1)
+        @status[status] += count
+      end
+
+      def ok?
+        ERROR_STATUS.each { |s| return false unless @status[s] == 0 }
+        @ping_solr && @ping_source
+      end
+    end
+  end
+end

data/lib/search_solr_tools/helpers/iso_to_solr.rb

@@ -1,3 +1,4 @@
+require_relative 'selectors'
 require 'nokogiri'
 
 module SearchSolrTools

data/lib/search_solr_tools/helpers/iso_to_solr_format.rb

@@ -1,7 +1,7 @@
 require 'date'
 
-require_relative './iso_namespaces'
-require_relative './solr_format'
+require_relative 'iso_namespaces'
+require_relative 'solr_format'
 
 module SearchSolrTools
   module Helpers

data/lib/search_solr_tools/helpers/ncdc_paleo_format.rb

@@ -1,8 +1,8 @@
 require 'date'
 
-require_relative './iso_namespaces'
-require_relative './solr_format'
-require_relative './iso_to_solr_format'
+require_relative 'iso_namespaces'
+require_relative 'solr_format'
+require_relative 'iso_to_solr_format'
 
 module SearchSolrTools
   module Helpers

data/lib/search_solr_tools/helpers/r2r_format.rb

@@ -1,6 +1,6 @@
-require_relative './iso_namespaces'
-require_relative './iso_to_solr_format'
-require_relative './solr_format'
+require_relative 'iso_namespaces'
+require_relative 'iso_to_solr_format'
+require_relative 'solr_format'
 
 module SearchSolrTools
   module Helpers

data/lib/search_solr_tools/helpers/selectors.rb

@@ -1,5 +1,4 @@
-require 'require_all'
-require_rel '../selectors'
+Dir[File.join(__dir__, '..', 'selectors', '*.rb')].each { |file| require file }
 
 module SearchSolrTools
   module Helpers

data/lib/search_solr_tools/helpers/solr_format.rb

@@ -2,6 +2,7 @@ require 'date'
 require 'iso8601'
 
 require_relative 'bounding_box_util'
+require_relative 'facet_configuration'
 
 module SearchSolrTools
   module Helpers

data/lib/search_solr_tools/helpers/tdar_format.rb

@@ -1,6 +1,6 @@
-require_relative './iso_namespaces'
-require_relative './iso_to_solr_format'
-require_relative './solr_format'
+require_relative 'iso_namespaces'
+require_relative 'iso_to_solr_format'
+require_relative 'solr_format'
 
 module SearchSolrTools
   module Helpers

data/lib/search_solr_tools/helpers/translate_spatial_coverage.rb

@@ -1,7 +1,7 @@
 require 'rgeo/geo_json'
 
-require 'search_solr_tools/helpers/bounding_box_util'
-require 'search_solr_tools/helpers/iso_to_solr_format'
+require_relative 'bounding_box_util'
+require_relative 'iso_to_solr_format'
 
 module SearchSolrTools
   module Helpers

data/lib/search_solr_tools/helpers/translate_temporal_coverage.rb

@@ -1,6 +1,6 @@
 require 'rgeo/geo_json'
 
-require 'search_solr_tools/helpers/solr_format'
+require_relative 'solr_format'
 
 module SearchSolrTools
   module Helpers

data/lib/search_solr_tools/helpers/usgs_format.rb

@@ -1,5 +1,5 @@
-require_relative './iso_namespaces'
-require_relative './iso_to_solr_format'
+require_relative 'iso_namespaces'
+require_relative 'iso_to_solr_format'
 
 module SearchSolrTools
   module Helpers

data/lib/search_solr_tools/selectors/adc.rb

@@ -1,4 +1,5 @@
-require 'search_solr_tools'
+require_relative '../helpers/solr_format'
+require_relative '../helpers/data_one_format'
 
 module SearchSolrTools
   module Selectors

data/lib/search_solr_tools/selectors/data_one.rb

@@ -1,4 +1,5 @@
-require 'search_solr_tools'
+require_relative '../helpers/data_one_format'
+require_relative '../helpers/solr_format'
 
 module SearchSolrTools
   module Selectors

data/lib/search_solr_tools/selectors/echo_iso.rb

@@ -1,4 +1,5 @@
-require 'search_solr_tools'
+require_relative '../helpers/solr_format'
+require_relative '../helpers/iso_to_solr_format'
 
 module SearchSolrTools
   module Selectors

data/lib/search_solr_tools/selectors/ices_iso.rb

@@ -1,4 +1,5 @@
-require 'search_solr_tools'
+require_relative '../helpers/solr_format'
+require_relative '../helpers/iso_to_solr_format'
 
 module SearchSolrTools
   module Selectors

data/lib/search_solr_tools/selectors/ncdc_paleo.rb

@@ -1,4 +1,5 @@
-require 'search_solr_tools'
+require_relative '../helpers/solr_format'
+require_relative '../helpers/ncdc_paleo_format'
 
 module SearchSolrTools
   module Selectors

data/lib/search_solr_tools/selectors/nmi.rb

@@ -1,4 +1,5 @@
-require 'search_solr_tools'
+require_relative '../helpers/solr_format'
+require_relative '../helpers/iso_to_solr_format'
 
 module SearchSolrTools
   module Selectors

data/lib/search_solr_tools/selectors/nodc_iso.rb

@@ -1,4 +1,5 @@
-require 'search_solr_tools'
+require_relative '../helpers/solr_format'
+require_relative '../helpers/iso_to_solr_format'
 
 module SearchSolrTools
   module Selectors

data/lib/search_solr_tools/selectors/pdc_iso.rb

@@ -1,4 +1,5 @@
-require 'search_solr_tools'
+require_relative '../helpers/solr_format'
+require_relative '../helpers/iso_to_solr_format'
 
 module SearchSolrTools
   module Selectors

data/lib/search_solr_tools/selectors/r2r.rb

@@ -1,4 +1,6 @@
-require 'search_solr_tools'
+require_relative '../helpers/solr_format'
+require_relative '../helpers/iso_to_solr_format'
+require_relative '../helpers/r2r_format'
 
 module SearchSolrTools
   module Selectors

data/lib/search_solr_tools/selectors/rda.rb

@@ -1,4 +1,5 @@
-require 'search_solr_tools'
+require_relative '../helpers/solr_format'
+require_relative '../helpers/iso_to_solr_format'
 
 module SearchSolrTools
   module Selectors

data/lib/search_solr_tools/selectors/tdar_opensearch.rb

@@ -1,4 +1,6 @@
-require 'search_solr_tools'
+require_relative '../helpers/solr_format'
+require_relative '../helpers/iso_to_solr_format'
+require_relative '../helpers/tdar_format'
 
 module SearchSolrTools
   module Selectors

data/lib/search_solr_tools/selectors/usgs_iso.rb

@@ -1,4 +1,6 @@
-require 'search_solr_tools'
+require_relative '../helpers/solr_format'
+require_relative '../helpers/iso_to_solr_format'
+require_relative '../helpers/usgs_format'
 
 module SearchSolrTools
   module Selectors

data/lib/search_solr_tools/translators/bcodmo_json.rb

@@ -4,6 +4,9 @@ require 'rgeo/geo_json'
 require 'rgeo/wkrep/wkt_parser'
 
 require 'search_solr_tools'
+require_relative '../helpers/solr_format'
+require_relative '../helpers/translate_temporal_coverage'
+require_relative '../helpers/translate_spatial_coverage'
 
 module SearchSolrTools
   module Translators