search_solr_tools 5.0.1 → 5.1.0.pre.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 763cb91da974c8cfa5841996b861f5a0492d950d3abf39fd62813af3a521ce22
-  data.tar.gz: d1d8ea0232c02f91f739baae6387a1ca71a03030962f0919904d03635b2d7ba2
+  metadata.gz: 6eb73f77a8b012afbb965dbeb800fd1a0fcdd3261711d2f4d218f35e0276f2c9
+  data.tar.gz: a04a061409ad6fca267e91fd21efb6d4ee010b75024e499eeb8f451be56ba0ea
 SHA512:
-  metadata.gz: 7c4bc5946b0a19f0634a28acfc6ff2282098ad3a730fb1060c4360f1a25a10edc78278c4ea123da688a4eeeaf62887ef6d86ec2ca24bebfd51f08783926d1681
-  data.tar.gz: 248141e1bc29f26b60795b1d4f73f9dda804ddbf1841ffbc490ca7690fe84fb2d1fb5b501c3f8df23cd8c9c11efd7dd66b65e0394114005f5e22c9b81b002781
+  metadata.gz: e39d629876795a63ed4802dc4cc334208300b82b42368eb7a00ecf9ad274b5af2444ac3a91204b80c50058ec1e8ecf3ea85f8aa35fc54fbd274620fef513d218
+  data.tar.gz: 367b9a206cb10f2b2b2285c2d6882c578c4df5eab99cbf9110f942ab7e98df9509849775ffe8f23d6222182bb4f35dd4ba23121d77a7228e9efa3b1895538fac

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## Unreleased
+
+  - Added a CLI method to "ping" the Solr and Source servers for a given 
+    data center.
+  - Updated the CLI harvest to return more useful error codes on failure.
+
 ## v5.0.1 (2020-07-02)
 
   - Bug fix: some requires weren't included that needed to be.

data/README.md

@@ -124,9 +124,9 @@ tagging, and publishing to RubyGems.
 |---------------------------|-------------|
 | `rake release:pre[false]` | Increase the current prerelease version number, push changes |
 | `rake release:pre[true]`  | Increase the current prerelease version number, publish release\* |
-| `rake release:none`       | Drop the prerelease version, publish release, then `pre[false]` (does a patch release) |
-| `rake release:minor`      | Increase the minor version number, publish release, then `pre[false]` |
-| `rake release:major`      | Increase the major version number, publish release, then `pre[false]` |
+| `rake release:none`       | Drop the prerelease version, publish release\*, then `pre[false]` (does a patch release) |
+| `rake release:minor`      | Increase the minor version number, publish release\*, then `pre[false]` |
+| `rake release:major`      | Increase the major version number, publish release\*, then `pre[false]` |
 
 \*"publish release" means each of the following occurs:
 

data/bin/search_solr_tools

@@ -4,6 +4,7 @@
 require 'search_solr_tools'
 require 'thor'
 
+# rubocop:disable Metrics/AbcSize
 class SolrHarvestCLI < Thor
   map %w[--version -v] => :__print_version
 
@@ -12,6 +13,48 @@ class SolrHarvestCLI < Thor
     puts SearchSolrTools::VERSION
   end
 
+  desc 'errcode', 'Get a description of the specified error exit code.  Leave empty to see all codes'
+  def errcode(code = -1)
+    codes = SearchSolrTools::Errors::HarvestError.describe_exit_code(code)
+
+    puts 'CODE | DESCRIPTION'
+    puts '-----+------------'
+    codes.each do |c, text|
+      puts format('%4<code>d | %<text>s', code: c, text: text)
+    end
+  end
+
+  desc 'ping', 'Ping the solr and harvesting endpoints related to the specified data center(s)'
+  option :data_center, type: :array, required: true
+  option :environment, required: true
+  def ping
+    solr_success = true
+    source_success = true
+    options[:data_center].each do |target|
+      begin
+        harvest_class = get_harvester_class(target)
+        harvester = harvest_class.new(options[:environment])
+        solr_status = harvester.ping_solr
+        source_status = harvester.ping_source
+      rescue StandardError => e
+        solr_status = false
+        source_status = false
+        puts "Error trying to ping for #{target}: #{e}"
+      end
+      solr_success &&= solr_status
+      source_success &&= source_status
+      puts "Target: #{target}, Solr ping OK? #{solr_status}, data center ping OK? #{source_status}"
+    end
+
+    ping_status = SearchSolrTools::Helpers::HarvestStatus.new(
+      SearchSolrTools::Helpers::HarvestStatus::PING_SOLR => solr_success,
+      SearchSolrTools::Helpers::HarvestStatus::PING_SOURCE => source_success
+    )
+    raise SearchSolrTools::Errors::HarvestError, ping_status unless ping_status.ok?
+  rescue SearchSolrTools::Errors::HarvestError => e
+    exit e.exit_code
+  end
+
   desc 'harvest', 'Harvest from the specified data centers'
   option :data_center, type: :array, required: true
   option :environment, required: true
@@ -22,10 +65,21 @@ class SolrHarvestCLI < Thor
       begin
         harvest_class = get_harvester_class(target)
         harvester = harvest_class.new(options[:environment], die_on_failure)
+        ping_status = SearchSolrTools::Helpers::HarvestStatus.new(
+          SearchSolrTools::Helpers::HarvestStatus::PING_SOLR => harvester.ping_solr,
+          SearchSolrTools::Helpers::HarvestStatus::PING_SOURCE => harvester.ping_source
+        )
+        raise SearchSolrTools::Errors::HarvestError, ping_status unless ping_status.ok?
+
         harvester.harvest_and_delete
+      rescue SearchSolrTools::Errors::HarvestError => e
+        puts "THERE WERE HARVEST STATUS ERRORS:\n#{e.message}"
+        exit e.exit_code
       rescue StandardError => e
+        # If it gets here, there is an error that we aren't expecting.
         puts "harvest failed for #{target}: #{e.message}"
-        raise e
+        puts e.backtrace
+        exit SearchSolrTools::Errors::HarvestError::ERRCODE_OTHER
       end
     end
   end
@@ -85,10 +139,12 @@ class SolrHarvestCLI < Thor
 
     def get_harvester_class(data_center_name)
       name = data_center_name.downcase.to_s
-      raise("Invalid data center #{name}") unless harvester_map.key?(name)
+      raise SearchSolrTools::Errors::HarvestError.new(nil, "Invalid data center #{name}") unless harvester_map.key?(name)
 
       harvester_map[name]
     end
   end
 end
+# rubocop:enable Metrics/AbcSize
+
 SolrHarvestCLI.start(ARGV)

data/lib/search_solr_tools.rb

@@ -2,6 +2,9 @@ require_relative 'search_solr_tools/config/environments'
 require_relative 'search_solr_tools/version'
 
 require_relative 'search_solr_tools/helpers/selectors'
+require_relative 'search_solr_tools/helpers/harvest_status'
+require_relative 'search_solr_tools/errors/harvest_error'
+
 %w( selectors harvesters translators ).each do |subdir|
   Dir[File.join(__dir__, 'search_solr_tools', subdir, '*.rb')].each { |file| require file }
 end

data/lib/search_solr_tools/errors/harvest_error.rb

@@ -0,0 +1,73 @@
+module SearchSolrTools
+  module Errors
+    class HarvestError < StandardError
+      ERRCODE_SOLR_PING = 1
+      ERRCODE_SOURCE_PING = 2
+      ERRCODE_SOURCE_NO_RESULTS = 4
+      ERRCODE_SOURCE_HARVEST_ERROR = 8
+      ERRCODE_DOCUMENT_INVALID = 16
+      ERRCODE_INGEST_ERROR = 32
+      ERRCODE_OTHER = 128
+
+      ERRCODE_DESC = {
+          ERRCODE_SOLR_PING => 'Solr instance did not return a successful ping',
+          ERRCODE_SOURCE_PING => 'Source to be harvested did not return a successful ping',
+          ERRCODE_SOURCE_NO_RESULTS => 'Source to be harvested returned no documents matching query',
+          ERRCODE_SOURCE_HARVEST_ERROR => 'One or more source documents returned an error when trying to retrieve or translate',
+          ERRCODE_DOCUMENT_INVALID => 'One or more documents to be harvested was invalid (malformed)',
+          ERRCODE_INGEST_ERROR => 'Solr returned an error trying to ingest one or more harvested documents',
+          ERRCODE_OTHER => 'General error code for non-harvest related issues'
+      }.freeze
+
+      # If code is -1, it means display all error codes
+      def self.describe_exit_code(code = -1)
+        code = code.to_i
+        code_list = []
+
+        # Loop through all bit-flag values
+        [128, 64, 32, 16, 8, 4, 2, 1].each do |k|
+          if code >= k || code == -1
+            code_list.prepend k
+            code -= k unless code == -1
+          end
+        end
+
+        codes = {}
+        code_list.each do |k|
+          next if code == -1 && !ERRCODE_DESC.keys.include?(k)  # skip INVALID CODE if showing all codes
+          codes[k] = ERRCODE_DESC.keys.include?(k) ? ERRCODE_DESC[k] : 'INVALID CODE NUMBER'
+        end
+
+        codes
+      end
+
+      def initialize(status, message=nil)
+        @status_data = status
+        @other_message = message
+      end
+
+      def exit_code
+        if @status_data.nil?
+          puts "OTHER ERROR REPORTED: #{@other_message}"
+          return ERRCODE_OTHER
+        end
+
+        code = 0
+        code += ERRCODE_SOLR_PING unless @status_data.ping_solr
+        code += ERRCODE_SOURCE_PING unless @status_data.ping_source
+        code += ERRCODE_SOURCE_NO_RESULTS if @status_data.documents_with_status(Helpers::HarvestStatus::HARVEST_NO_DOCS).size > 0
+        code += ERRCODE_SOURCE_HARVEST_ERROR if @status_data.documents_with_status(Helpers::HarvestStatus::HARVEST_FAILURE).size > 0
+        code += ERRCODE_DOCUMENT_INVALID if @status_data.documents_with_status(Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC).size > 0
+        code += ERRCODE_INGEST_ERROR if @status_data.documents_with_status(Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR).size > 0
+
+        code = ERRCODE_OTHER if code == 0
+
+        code
+      end
+
+      def message
+        self.class.describe_exit_code(exit_code).map{|c,v| v}.join("\n")
+      end
+    end
+  end
+end

data/lib/search_solr_tools/harvesters/auto_suggest.rb

@@ -46,8 +46,11 @@ module SearchSolrTools
       end
 
       def add_documents_to_solr(add_docs)
-        if insert_solr_doc add_docs, Base::JSON_CONTENT_TYPE, @env_settings[:auto_suggest_collection_name]
+        status = insert_solr_doc add_docs, Base::JSON_CONTENT_TYPE, @env_settings[:auto_suggest_collection_name]
+
+        if status == Helpers::HarvestStatus::INGEST_OK
           puts "Added #{add_docs.size} auto suggest documents in one commit"
+          return Helpers::HarvestStatus.new(Helpers::HarvestStatus::INGEST_OK => add_docs)
         else
           puts "Failed adding #{add_docs.size} documents in single commit, retrying one by one"
           new_add_docs = []

data/lib/search_solr_tools/harvesters/base.rb

@@ -39,10 +39,40 @@ module SearchSolrTools
         url
       end
 
+      # Ping the Solr instance to ensure that it's running.
+      # The ping query is specified to manually check the title, as it's possible
+      # there is no "default" query in the solr instance.
+      def ping_solr(core = SolrEnvironments[@environment][:collection_name])
+        url = solr_url + "/#{core}/admin/ping?df=title"
+        success = false
+
+        # Some docs will cause solr to time out during the POST
+        begin
+          RestClient.get(url) do |response, _request, _result|
+            success = response.code == 200
+            puts "Error in ping request: #{response.body}" unless success
+          end
+        rescue => e
+          puts "Rest exception while pinging Solr: #{e}"
+        end
+        success
+      end
+
+      # This should be overridden by child classes to implement the ability
+      # to "ping" the dataset.  Returns true if the ping is successful (or, as
+      # in this default, no ping method was defined)
+      def ping_source
+        puts "Harvester does not have ping method defined, assuming true"
+        true
+      end
+
       def harvest_and_delete(harvest_method, delete_constraints, solr_core = SolrEnvironments[@environment][:collection_name])
         start_time = Time.now.utc.iso8601
-        harvest_method.call
+
+        harvest_status = harvest_method.call
         delete_old_documents start_time, delete_constraints, solr_core
+
+        harvest_status
       end
 
       def delete_old_documents(timestamp, constraints, solr_core, force = false)
@@ -83,21 +113,31 @@ module SearchSolrTools
       def insert_solr_docs(docs, content_type = XML_CONTENT_TYPE, core = SolrEnvironments[@environment][:collection_name])
         success = 0
         failure = 0
+
+        status = Helpers::HarvestStatus.new
+
         docs.each do |doc|
-          insert_solr_doc(doc, content_type, core) ? success += 1 : failure += 1
+          doc_status = insert_solr_doc(doc, content_type, core) #? success += 1 : failure += 1
+          status.record_document_status(doc, doc_status)
+          doc_status == Helpers::HarvestStatus::INGEST_OK ? success += 1 : failure += 1
         end
         puts "#{success} document#{success == 1 ? '' : 's'} successfully added to Solr."
         puts "#{failure} document#{failure == 1 ? '' : 's'} not added to Solr."
-        fail 'Some documents failed to be inserted into Solr' if failure > 0
+
+        status
       end
 
+      # TODO Need to return a specific type of failure:
+      #   - Bad record content identified and no ingest attempted
+      #   - Solr tries to ingest document and fails (bad content not detected prior to ingest)
+      #   - Solr cannot insert document for reasons other than the document structure and content.
       def insert_solr_doc(doc, content_type = XML_CONTENT_TYPE, core = SolrEnvironments[@environment][:collection_name])
         url = solr_url + "/#{core}/update?commit=true"
-        success = false
+        status = Helpers::HarvestStatus::INGEST_OK
 
         # Some of the docs will cause Solr to crash - CPU goes to 195% with `top` and it
         # doesn't seem to recover.
-        return success if content_type == XML_CONTENT_TYPE && !doc_valid?(doc)
+        return Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC if content_type == XML_CONTENT_TYPE && !doc_valid?(doc)
 
         doc_serialized = get_serialized_doc(doc, content_type)
 
@@ -105,13 +145,18 @@ module SearchSolrTools
         begin
           RestClient.post(url, doc_serialized, content_type: content_type) do |response, _request, _result|
             success = response.code == 200
-            puts "Error for #{doc_serialized}\n\n response: #{response.body}" unless success
+            unless success
+              puts "Error for #{doc_serialized}\n\n response: #{response.body}"
+              status = Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR
+            end
           end
         rescue => e
+          # TODO Need to provide more detail re: this failure so we know whether to
+          #  exit the job with a status != 0
           puts "Rest exception while POSTing to Solr: #{e}, for doc: #{doc_serialized}"
+          status = Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR
         end
-
-        success
+        status
       end
 
       def get_serialized_doc(doc, content_type)
@@ -124,7 +169,7 @@ module SearchSolrTools
         end
       end
 
-      # Get results from some ISO end point specified in the query string
+      # Get results from an end point specified in the request_url
       def get_results(request_url, metadata_path, content_type = 'application/xml')
         timeout = 300
         retries_left = 3
@@ -140,6 +185,9 @@ module SearchSolrTools
 
           retry if retries_left > 0
 
+          # TODO - Do we really need this "die_on_failure" anymore?  The empty return
+          #  will cause the "No Documents" error to be thrown in the harvester class
+          #  now, so it will pretty much always "die on failure"
           raise e if @die_on_failure
           return
         end

data/lib/search_solr_tools/harvesters/nsidc_json.rb

@@ -3,6 +3,7 @@ require 'rest-client'
 
 require 'search_solr_tools'
 
+
 module SearchSolrTools
   module Harvesters
     # Harvests data from NSIDC OAI and inserts it into Solr after it has been translated
@@ -13,6 +14,17 @@ module SearchSolrTools
         Helpers::FacetConfiguration.import_bin_configuration(env)
       end
 
+      def ping_source
+        begin
+          RestClient.options(nsidc_json_url) do |response, _request, _result|
+            return response.code == 200
+          end
+        rescue => e
+          puts "Error trying to get options for #{nsidc_json_url} (ping)"
+        end
+        false
+      end
+
       def harvest_and_delete
         puts "Running harvest of NSIDC catalog from #{nsidc_json_url}"
         super(method(:harvest_nsidc_json_into_solr), "data_centers:\"#{Helpers::SolrFormat::DATA_CENTER_NAMES[:NSIDC][:long_name]}\"")
@@ -22,8 +34,21 @@ module SearchSolrTools
       # this is the main entry point for the class
       def harvest_nsidc_json_into_solr
         result = docs_with_translated_entries_from_nsidc
-        insert_solr_docs result[:add_docs], Base::JSON_CONTENT_TYPE
-        fail 'Failed to harvest and insert some authoritative IDs' if result[:failure_ids].length > 0
+
+        status = insert_solr_docs result[:add_docs], Base::JSON_CONTENT_TYPE
+
+        status.record_document_status('harvest', Helpers::HarvestStatus::HARVEST_NO_DOCS) if result[:num_docs] == 0
+        status.record_multiple_document_status(result[:failure_ids],
+                                               Helpers::HarvestStatus::HARVEST_FAILURE) if result[:failure_ids].length > 0
+
+        raise Errors::HarvestError, status unless status.ok?
+      rescue Errors::HarvestError => e
+        raise e
+      rescue StandardError => e
+        puts "An unexpected exception occurred while trying to harvest or insert: #{e}"
+        puts e.backtrace
+        status = Helpers::HarvestStatus.new(Helpers::HarvestStatus::OTHER_ERROR => e)
+        raise Errors::HarvestError, status
       end
 
       def nsidc_json_url
@@ -33,7 +58,7 @@ module SearchSolrTools
       def result_ids_from_nsidc
         url = SolrEnvironments[@environment][:nsidc_dataset_metadata_url] +
               SolrEnvironments[@environment][:nsidc_oai_identifiers_url]
-        get_results url, '//xmlns:identifier'
+        get_results(url, '//xmlns:identifier') || []
       end
 
       # Fetch a JSON representation of a dataset's metadata
@@ -48,7 +73,8 @@ module SearchSolrTools
         docs = []
         failure_ids = []
 
-        result_ids_from_nsidc.each do |r|
+        all_docs = result_ids_from_nsidc
+        all_docs.each do |r|
           # Each result looks like:
           # oai:nsidc.org/AE_L2A
           id = r.text.split('/').last
@@ -60,7 +86,7 @@ module SearchSolrTools
           end
         end
 
-        { add_docs: docs, failure_ids: failure_ids }
+        { num_docs: all_docs.size, add_docs: docs, failure_ids: failure_ids }
       end
     end
   end

data/lib/search_solr_tools/helpers/harvest_status.rb

@@ -0,0 +1,66 @@
+module SearchSolrTools
+  module Helpers
+    class HarvestStatus
+
+      INGEST_OK = :ok
+      HARVEST_NO_DOCS = :harvest_none
+      HARVEST_FAILURE = :harvest_fail
+      INGEST_ERR_INVALID_DOC = :invalid
+      INGEST_ERR_SOLR_ERROR = :solr_error
+      OTHER_ERROR = :other
+      PING_SOLR = :ping_solr  # used for initialize only
+      PING_SOURCE = :ping_source  # used for initialize only
+
+      ERROR_STATUS = [HARVEST_NO_DOCS, HARVEST_FAILURE, INGEST_ERR_INVALID_DOC, INGEST_ERR_SOLR_ERROR, OTHER_ERROR]
+
+      # init_info is an optional hash that contains the various status keys and the documents to
+      # associate with them
+      def initialize(init_info={})
+        @status = { INGEST_OK => [] }
+        @ping_solr = true
+        @ping_source = true
+        ERROR_STATUS.each { |s| @status[s] = [] }
+
+        init_info.each do |key, docs|
+          @status[key] = docs if @status.include? key
+        end
+
+        @ping_solr = init_info[PING_SOLR] if init_info.include? PING_SOLR
+        @ping_source = init_info[PING_SOURCE] if init_info.include? PING_SOURCE
+      end
+
+      def record_multiple_document_status(documents, doc_status)
+        documents.each { |d| record_document_status d, doc_status }
+      end
+
+      def record_document_status(document, doc_status)
+        @status[doc_status] << document
+      end
+
+      def ping_solr=(newval)
+        @ping_solr = newval
+      end
+
+      def ping_source=(newval)
+        @ping_source = newval
+      end
+
+      def ok?
+        ERROR_STATUS.each { |s| return false unless @status[s].empty? }
+        @ping_solr && @ping_source
+      end
+
+      def ping_solr
+        @ping_solr
+      end
+
+      def ping_source
+        @ping_source
+      end
+
+      def documents_with_status(doc_status)
+        @status[doc_status]
+      end
+    end
+  end
+end

data/lib/search_solr_tools/version.rb

@@ -1,3 +1,3 @@
 module SearchSolrTools
-  VERSION = '5.0.1.release.1'
+  VERSION = '5.1.0.pre.2'
 end

data/search_solr_tools.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
     metadata into a working solr instance.
   EOF
   spec.homepage      = 'https://github.com/nsidc/search-solr-tools'
-  spec.license       = 'GNU GPL Version 3'
+  spec.license       = 'GPL-3.0-or-later'
 
   spec.files         = `git ls-files -z #{gem_files}`.split("\x0")
   spec.executables   = spec.files.grep(/^bin\//) { |f| File.basename(f) }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: search_solr_tools
 version: !ruby/object:Gem::Version
-  version: 5.0.1
+  version: 5.1.0.pre.1
 platform: ruby
 authors:
 - Chris Chalstrom
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-02 00:00:00.000000000 Z
+date: 2020-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi-geos
@@ -292,6 +292,7 @@ files:
 - lib/search_solr_tools.rb
 - lib/search_solr_tools/config/environments.rb
 - lib/search_solr_tools/config/environments.yaml
+- lib/search_solr_tools/errors/harvest_error.rb
 - lib/search_solr_tools/harvesters/adc.rb
 - lib/search_solr_tools/harvesters/ade_auto_suggest.rb
 - lib/search_solr_tools/harvesters/auto_suggest.rb
@@ -317,6 +318,7 @@ files:
 - lib/search_solr_tools/helpers/csw_iso_query_builder.rb
 - lib/search_solr_tools/helpers/data_one_format.rb
 - lib/search_solr_tools/helpers/facet_configuration.rb
+- lib/search_solr_tools/helpers/harvest_status.rb
 - lib/search_solr_tools/helpers/iso_namespaces.rb
 - lib/search_solr_tools/helpers/iso_to_solr.rb
 - lib/search_solr_tools/helpers/iso_to_solr_format.rb
@@ -349,7 +351,7 @@ files:
 - search_solr_tools.gemspec
 homepage: https://github.com/nsidc/search-solr-tools
 licenses:
-- GNU GPL Version 3
+- GPL-3.0-or-later
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -362,9 +364,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.6.5
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.8
 signing_key: