search_solr_tools 4.2.0 → 5.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5929841e7bf672bf499a73bbfd49a4f7b13db0238cfd57c3705d59513351fc30
-  data.tar.gz: f4a8809af8058bb43f24a0a0842b89e5158ca96ae16145d65f63dd8a6030f8bd
+  metadata.gz: a961c7f8cdb3a126f31ba685db351337dda8816f64de94b82aad774f53b49a0c
+  data.tar.gz: bcadb76963b19f66567c3e5a233dd599a66d642073eb5c043b3fc5653ce2502f
 SHA512:
-  metadata.gz: 867c37d6e1b39d2a7de7a001c0aec4cc0891338f0d3504e7de8062b3ff88aeef1e8794f312f97b038bd4b1a4936e58e5629ea97907c417d13e9ead01d502cfb4
-  data.tar.gz: d0eb4c0cdd073d4c34c8460a72e767347d52dcc0d03de007c829e7d6b2a20075e8013d1764db39e74e4d62a62e43537599b28d98b8ab0cbe163499663b578e64
+  metadata.gz: 1a238ac9143e968252c4f37474c99ed38a6920160f9f3d8ba8753b72848dbf7152ba0bc0b142298141eae6693102481a34f20d7c461e869b1a1606b7a3fcb471
+  data.tar.gz: 0ca3f51c62c0683d58652928ea2bf8a09987cee68d8af691ee523186e1d3ce7dc0f4bec0e294cdb82206eb8ebb5d2b6ad6085bed651b6f0944f979cecb0875bb

data/CHANGELOG.md

@@ -1,9 +1,35 @@
-## v4.1.0 (2019-08-12)
+## v5.1.0 (2020-07-23)
+
+  - Added a CLI method to "ping" the Solr and Source servers for a given 
+    data center.
+  - Added a CLI method "errcode" to get information about the various 
+    error codes that may be returned during harvest
+  - Updated the CLI harvest to return more useful error codes on failure.
+
+## v5.0.1 (2020-07-02)
+
+  - Bug fix: some requires weren't included that needed to be.
+
+## v5.0.0 (2020-07-02)
+
+  - Update Ruby to 2.6.5, update gem dependencies to more recent version.
+  - Updates to correspond with an update to Solr 8.5.2
+
+## v4.2.1 (2019-08-13)
+
+  - Patch release to include updated CHANGELOG.
+
+## v4.2.0 (2019-08-12)
 
   - Update dataset-catalog-services URL to only fetch current (*not* retired)
     metadata records.
   - Add a few more gem release notes to README.
 
+Note: v4.1.0 was prematurely released and, in theory, yanked. However, on the
+second try at publishing 4.1.0, Rubygems complained about the attempt to
+republish a gem. The version was therefore bumped again to 4.2.0 as the path of
+least resistance to a successful publish. v4.1.0 should not be used.
+
 ## v4.0.1 (2019-07-08)
 
   - Update CHANGELOG and release instructions.

data/README.md

@@ -61,7 +61,7 @@ Once you have the gem built in the project directory, install the utility:
 
 ### Requirements
 
-* Ruby > 2.0.0
+* Ruby > 2.6.5
 * [Bundler](http://bundler.io/)
 * Requirements for nokogiri:
     * [libxml2/libxml2-dev](http://xmlsoft.org/)
@@ -105,7 +105,7 @@ Please be sure to run them in the `bundle exec` context if you're utilizing bund
 
 Requirements:
 
-* Ruby > 2.0.0
+* Ruby > 2.6.5
 * [Bundler](http://bundler.io/)
 * [Gem Release](https://github.com/svenfuchs/gem-release)
 * [Rake](https://github.com/ruby/rake)
@@ -124,9 +124,9 @@ tagging, and publishing to RubyGems.
 |---------------------------|-------------|
 | `rake release:pre[false]` | Increase the current prerelease version number, push changes |
 | `rake release:pre[true]`  | Increase the current prerelease version number, publish release\* |
-| `rake release:none`       | Drop the prerelease version, publish release, then `pre[false]` (does a patch release) |
-| `rake release:minor`      | Increase the minor version number, publish release, then `pre[false]` |
-| `rake release:major`      | Increase the major version number, publish release, then `pre[false]` |
+| `rake release:none`       | Drop the prerelease version, publish release\*, then `pre[false]` (does a patch release) |
+| `rake release:minor`      | Increase the minor version number, publish release\*, then `pre[false]` |
+| `rake release:major`      | Increase the major version number, publish release\*, then `pre[false]` |
 
 \*"publish release" means each of the following occurs:
 
@@ -150,7 +150,7 @@ order to publish a new version of the gem to Rubygems. To get the lastest API ke
 ### SOLR
 
 To harvest data utilizing the gem, you will need an installed instance of [Solr
-8.1.1](https://lucene.apache.org/solr/guide/)
+8.5.3](https://lucene.apache.org/solr/guide/)
 
 #### NSIDC
 

data/bin/search_solr_tools

@@ -4,6 +4,7 @@
 require 'search_solr_tools'
 require 'thor'
 
+# rubocop:disable Metrics/AbcSize
 class SolrHarvestCLI < Thor
   map %w[--version -v] => :__print_version
 
@@ -12,6 +13,48 @@ class SolrHarvestCLI < Thor
     puts SearchSolrTools::VERSION
   end
 
+  desc 'errcode CODE', 'Print all exit codes bundled in CODE.  Omit CODE to print all codes'
+  def errcode(code = -1)
+    codes = SearchSolrTools::Errors::HarvestError.describe_exit_code(code)
+
+    puts 'CODE | DESCRIPTION'
+    puts '-----+------------'
+    codes.each do |c, text|
+      puts format('%4<code>d | %<text>s', code: c, text: text)
+    end
+  end
+
+  desc 'ping', 'Ping the solr and harvesting endpoints related to the specified data center(s)'
+  option :data_center, type: :array, required: true
+  option :environment, required: true
+  def ping
+    solr_success = true
+    source_success = true
+    options[:data_center].each do |target|
+      begin
+        harvest_class = get_harvester_class(target)
+        harvester = harvest_class.new(options[:environment])
+        solr_status = harvester.ping_solr
+        source_status = harvester.ping_source
+      rescue StandardError => e
+        solr_status = false
+        source_status = false
+        puts "Error trying to ping for #{target}: #{e}"
+      end
+      solr_success &&= solr_status
+      source_success &&= source_status
+      puts "Target: #{target}, Solr ping OK? #{solr_status}, data center ping OK? #{source_status}"
+    end
+
+    ping_status = SearchSolrTools::Helpers::HarvestStatus.new(
+      SearchSolrTools::Helpers::HarvestStatus::PING_SOLR => solr_success,
+      SearchSolrTools::Helpers::HarvestStatus::PING_SOURCE => source_success
+    )
+    raise SearchSolrTools::Errors::HarvestError, ping_status unless ping_status.ok?
+  rescue SearchSolrTools::Errors::HarvestError => e
+    exit e.exit_code
+  end
+
   desc 'harvest', 'Harvest from the specified data centers'
   option :data_center, type: :array, required: true
   option :environment, required: true
@@ -22,10 +65,21 @@ class SolrHarvestCLI < Thor
       begin
         harvest_class = get_harvester_class(target)
         harvester = harvest_class.new(options[:environment], die_on_failure)
+        ping_status = SearchSolrTools::Helpers::HarvestStatus.new(
+          SearchSolrTools::Helpers::HarvestStatus::PING_SOLR => harvester.ping_solr,
+          SearchSolrTools::Helpers::HarvestStatus::PING_SOURCE => harvester.ping_source
+        )
+        raise SearchSolrTools::Errors::HarvestError, ping_status unless ping_status.ok?
+
         harvester.harvest_and_delete
+      rescue SearchSolrTools::Errors::HarvestError => e
+        puts "THERE WERE HARVEST STATUS ERRORS:\n#{e.message}"
+        exit e.exit_code
       rescue StandardError => e
+        # If it gets here, there is an error that we aren't expecting.
         puts "harvest failed for #{target}: #{e.message}"
-        raise e
+        puts e.backtrace
+        exit SearchSolrTools::Errors::HarvestError::ERRCODE_OTHER
       end
     end
   end
@@ -85,10 +139,12 @@ class SolrHarvestCLI < Thor
 
     def get_harvester_class(data_center_name)
       name = data_center_name.downcase.to_s
-      raise("Invalid data center #{name}") unless harvester_map.key?(name)
+      raise SearchSolrTools::Errors::HarvestError.new(nil, "Invalid data center #{name}") unless harvester_map.key?(name)
 
       harvester_map[name]
     end
   end
 end
+# rubocop:enable Metrics/AbcSize
+
 SolrHarvestCLI.start(ARGV)

data/lib/search_solr_tools.rb

@@ -1,8 +1,10 @@
-require 'require_all'
-require_relative './search_solr_tools/config/environments'
-require_relative './search_solr_tools/version'
+require_relative 'search_solr_tools/config/environments'
+require_relative 'search_solr_tools/version'
 
-require_rel './search_solr_tools/helpers'
-require_rel './search_solr_tools/selectors'
-require_rel './search_solr_tools/harvesters'
-require_rel './search_solr_tools/translators'
+require_relative 'search_solr_tools/helpers/selectors'
+require_relative 'search_solr_tools/helpers/harvest_status'
+require_relative 'search_solr_tools/errors/harvest_error'
+
+%w( selectors harvesters translators ).each do |subdir|
+  Dir[File.join(__dir__, 'search_solr_tools', subdir, '*.rb')].each { |file| require file }
+end

data/lib/search_solr_tools/errors/harvest_error.rb

@@ -0,0 +1,88 @@
+module SearchSolrTools
+  module Errors
+    class HarvestError < StandardError
+      ERRCODE_SOLR_PING = 1
+      ERRCODE_SOURCE_PING = 2
+      ERRCODE_SOURCE_NO_RESULTS = 4
+      ERRCODE_SOURCE_HARVEST_ERROR = 8
+      ERRCODE_DOCUMENT_INVALID = 16
+      ERRCODE_INGEST_ERROR = 32
+      ERRCODE_OTHER = 128
+
+      ERRCODE_DESC = {
+          ERRCODE_SOLR_PING => 'Solr instance did not return a successful ping',
+          ERRCODE_SOURCE_PING => 'Source to be harvested did not return a successful ping',
+          ERRCODE_SOURCE_NO_RESULTS => 'Source to be harvested returned no documents matching query',
+          ERRCODE_SOURCE_HARVEST_ERROR => 'One or more source documents returned an error when trying to retrieve or translate',
+          ERRCODE_DOCUMENT_INVALID => 'One or more documents to be harvested was invalid (malformed)',
+          ERRCODE_INGEST_ERROR => 'Solr returned an error trying to ingest one or more harvested documents',
+          ERRCODE_OTHER => 'General error code for non-harvest related issues'
+      }.freeze
+
+      PING_ERRCODE_MAP = {
+        'ping_solr' => ERRCODE_SOLR_PING,
+        'ping_source' => ERRCODE_SOURCE_PING,
+      }
+
+      STATUS_ERRCODE_MAP = {
+          Helpers::HarvestStatus::HARVEST_NO_DOCS => ERRCODE_SOURCE_NO_RESULTS,
+          Helpers::HarvestStatus::HARVEST_FAILURE => ERRCODE_SOURCE_HARVEST_ERROR,
+          Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC => ERRCODE_DOCUMENT_INVALID,
+          Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR => ERRCODE_INGEST_ERROR,
+          Helpers::HarvestStatus::OTHER_ERROR => ERRCODE_OTHER
+      }.freeze
+
+      # If code is -1, it means display all error codes
+      def self.describe_exit_code(code = -1)
+        code = code.to_i
+        code_list = []
+
+        # Loop through all bit-flag values
+        [128, 64, 32, 16, 8, 4, 2, 1].each do |k|
+          if code >= k || code == -1
+            code_list.prepend k
+            code -= k unless code == -1
+          end
+        end
+
+        codes = {}
+        code_list.each do |k|
+          next if code == -1 && !ERRCODE_DESC.keys.include?(k)  # skip INVALID CODE if showing all codes
+          codes[k] = ERRCODE_DESC.keys.include?(k) ? ERRCODE_DESC[k] : 'INVALID CODE NUMBER'
+        end
+
+        codes
+      end
+
+      def initialize(status, message=nil)
+        @status_data = status
+        @other_message = message
+      end
+
+      def exit_code
+        if @status_data.nil?
+          puts "OTHER ERROR REPORTED: #{@other_message}"
+          return ERRCODE_OTHER
+        end
+
+        puts "EXIT CODE STATUS:\n#{@status_data.status}"
+
+        code = 0
+        code += ERRCODE_SOLR_PING unless @status_data.ping_solr
+        code += ERRCODE_SOURCE_PING unless @status_data.ping_source
+        code += ERRCODE_SOURCE_NO_RESULTS if @status_data.status[Helpers::HarvestStatus::HARVEST_NO_DOCS] > 0
+        code += ERRCODE_SOURCE_HARVEST_ERROR if @status_data.status[Helpers::HarvestStatus::HARVEST_FAILURE] > 0
+        code += ERRCODE_DOCUMENT_INVALID if @status_data.status[Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC] > 0
+        code += ERRCODE_INGEST_ERROR if @status_data.status[Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR] > 0
+
+        code = ERRCODE_OTHER if code == 0
+
+        code
+      end
+
+      def message
+        self.class.describe_exit_code(exit_code).map{|c,v| v}.join("\n")
+      end
+    end
+  end
+end

data/lib/search_solr_tools/harvesters/adc.rb

@@ -1,3 +1,5 @@
+require_relative 'base'
+
 module SearchSolrTools
   module Harvesters
     class Adc < Base

data/lib/search_solr_tools/harvesters/ade_auto_suggest.rb

@@ -1,3 +1,5 @@
+require_relative 'auto_suggest'
+
 module SearchSolrTools
   module Harvesters
     class AdeAutoSuggest < AutoSuggest

data/lib/search_solr_tools/harvesters/auto_suggest.rb

@@ -1,3 +1,4 @@
+require_relative 'base'
 require 'json'
 require 'rest-client'
 
@@ -45,8 +46,11 @@ module SearchSolrTools
       end
 
       def add_documents_to_solr(add_docs)
-        if insert_solr_doc add_docs, Base::JSON_CONTENT_TYPE, @env_settings[:auto_suggest_collection_name]
+        status = insert_solr_doc add_docs, Base::JSON_CONTENT_TYPE, @env_settings[:auto_suggest_collection_name]
+
+        if status == Helpers::HarvestStatus::INGEST_OK
           puts "Added #{add_docs.size} auto suggest documents in one commit"
+          return Helpers::HarvestStatus.new(Helpers::HarvestStatus::INGEST_OK => add_docs)
         else
           puts "Failed adding #{add_docs.size} documents in single commit, retrying one by one"
           new_add_docs = []

data/lib/search_solr_tools/harvesters/base.rb

@@ -5,6 +5,12 @@ require 'rest-client'
 require 'rsolr'
 require 'time'
 
+require 'search_solr_tools'
+require_relative '../helpers/iso_namespaces'
+require_relative '../helpers/solr_format'
+require_relative '../helpers/iso_to_solr'
+
+
 module SearchSolrTools
   module Harvesters
     # base class for solr harvesters
@@ -33,17 +39,47 @@ module SearchSolrTools
         url
       end
 
+      # Ping the Solr instance to ensure that it's running.
+      # The ping query is specified to manually check the title, as it's possible
+      # there is no "default" query in the solr instance.
+      def ping_solr(core = SolrEnvironments[@environment][:collection_name])
+        url = solr_url + "/#{core}/admin/ping?df=title"
+        success = false
+
+        # Some docs will cause solr to time out during the POST
+        begin
+          RestClient.get(url) do |response, _request, _result|
+            success = response.code == 200
+            puts "Error in ping request: #{response.body}" unless success
+          end
+        rescue => e
+          puts "Rest exception while pinging Solr: #{e}"
+        end
+        success
+      end
+
+      # This should be overridden by child classes to implement the ability
+      # to "ping" the data center.  Returns true if the ping is successful (or, as
+      # in this default, no ping method was defined)
+      def ping_source
+        puts "Harvester does not have ping method defined, assuming true"
+        true
+      end
+
       def harvest_and_delete(harvest_method, delete_constraints, solr_core = SolrEnvironments[@environment][:collection_name])
         start_time = Time.now.utc.iso8601
-        harvest_method.call
+
+        harvest_status = harvest_method.call
         delete_old_documents start_time, delete_constraints, solr_core
+
+        harvest_status
       end
 
       def delete_old_documents(timestamp, constraints, solr_core, force = false)
         constraints = sanitize_data_centers_constraints(constraints)
         delete_query = "last_update:[* TO #{timestamp}] AND #{constraints}"
         solr = RSolr.connect url: solr_url + "/#{solr_core}"
-        unchanged_count = (solr.get 'select', params: { q: delete_query, rows: 0 })['response']['numFound'].to_i
+        unchanged_count = (solr.get 'select', params: { wt: :ruby, q: delete_query, rows: 0 })['response']['numFound'].to_i
         if unchanged_count == 0
           puts "All documents were updated after #{timestamp}, nothing to delete"
         else
@@ -61,7 +97,7 @@ module SearchSolrTools
       end
 
       def remove_documents(solr, delete_query, constraints, force, numfound)
-        all_response_count = (solr.get 'select', params: { q: constraints, rows: 0 })['response']['numFound']
+        all_response_count = (solr.get 'select', params: { wt: :ruby, q: constraints, rows: 0 })['response']['numFound']
         if force || (numfound / all_response_count.to_f < DELETE_DOCUMENTS_RATIO)
           puts "Deleting #{numfound} documents for #{constraints}"
           solr.delete_by_query delete_query
@@ -77,21 +113,31 @@ module SearchSolrTools
       def insert_solr_docs(docs, content_type = XML_CONTENT_TYPE, core = SolrEnvironments[@environment][:collection_name])
         success = 0
         failure = 0
+
+        status = Helpers::HarvestStatus.new
+
         docs.each do |doc|
-          insert_solr_doc(doc, content_type, core) ? success += 1 : failure += 1
+          doc_status = insert_solr_doc(doc, content_type, core)
+          status.record_status doc_status
+          doc_status == Helpers::HarvestStatus::INGEST_OK ? success += 1 : failure += 1
         end
         puts "#{success} document#{success == 1 ? '' : 's'} successfully added to Solr."
         puts "#{failure} document#{failure == 1 ? '' : 's'} not added to Solr."
-        fail 'Some documents failed to be inserted into Solr' if failure > 0
+
+        status
       end
 
+      # TODO Need to return a specific type of failure:
+      #   - Bad record content identified and no ingest attempted
+      #   - Solr tries to ingest document and fails (bad content not detected prior to ingest)
+      #   - Solr cannot insert document for reasons other than the document structure and content.
       def insert_solr_doc(doc, content_type = XML_CONTENT_TYPE, core = SolrEnvironments[@environment][:collection_name])
         url = solr_url + "/#{core}/update?commit=true"
-        success = false
+        status = Helpers::HarvestStatus::INGEST_OK
 
         # Some of the docs will cause Solr to crash - CPU goes to 195% with `top` and it
         # doesn't seem to recover.
-        return success if content_type == XML_CONTENT_TYPE && !doc_valid?(doc)
+        return Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC if content_type == XML_CONTENT_TYPE && !doc_valid?(doc)
 
         doc_serialized = get_serialized_doc(doc, content_type)
 
@@ -99,13 +145,18 @@ module SearchSolrTools
         begin
           RestClient.post(url, doc_serialized, content_type: content_type) do |response, _request, _result|
             success = response.code == 200
-            puts "Error for #{doc_serialized}\n\n response: #{response.body}" unless success
+            unless success
+              puts "Error for #{doc_serialized}\n\n response: #{response.body}"
+              status = Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR
+            end
           end
         rescue => e
+          # TODO Need to provide more detail re: this failure so we know whether to
+          #  exit the job with a status != 0
           puts "Rest exception while POSTing to Solr: #{e}, for doc: #{doc_serialized}"
+          status = Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR
         end
-
-        success
+        status
       end
 
       def get_serialized_doc(doc, content_type)
@@ -118,7 +169,7 @@ module SearchSolrTools
         end
       end
 
-      # Get results from some ISO end point specified in the query string
+      # Get results from an end point specified in the request_url
       def get_results(request_url, metadata_path, content_type = 'application/xml')
         timeout = 300
         retries_left = 3
@@ -134,6 +185,9 @@ module SearchSolrTools
 
           retry if retries_left > 0
 
+          # TODO - Do we really need this "die_on_failure" anymore?  The empty return
+          #  will cause the "No Documents" error to be thrown in the harvester class
+          #  now, so it will pretty much always "die on failure"
           raise e if @die_on_failure
           return
         end