search_solr_tools 5.1.0.pre.1 → 5.1.0.pre.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6eb73f77a8b012afbb965dbeb800fd1a0fcdd3261711d2f4d218f35e0276f2c9
-  data.tar.gz: a04a061409ad6fca267e91fd21efb6d4ee010b75024e499eeb8f451be56ba0ea
+  metadata.gz: 1b2767fe90ced0e512cd651820392f512e0d5ebb5bd5d1ec8d8044d01c90966f
+  data.tar.gz: 3c01a52136449c3281534f0353091fb607d2a28c6595392b035e14bae042c62c
 SHA512:
-  metadata.gz: e39d629876795a63ed4802dc4cc334208300b82b42368eb7a00ecf9ad274b5af2444ac3a91204b80c50058ec1e8ecf3ea85f8aa35fc54fbd274620fef513d218
-  data.tar.gz: 367b9a206cb10f2b2b2285c2d6882c578c4df5eab99cbf9110f942ab7e98df9509849775ffe8f23d6222182bb4f35dd4ba23121d77a7228e9efa3b1895538fac
+  metadata.gz: f478d8680f4ef4dd3acc5958906d65d1470f197e999f6206208a315051e3017ce54c32cc92730b78472e953b68f988bf1fea6681c2c3903a46ffe4a8bf4dbde6
+  data.tar.gz: 31c5f916ae86a7e2878598568e5de3caa5dade77378788a7721360bf742f8d433b8b3db7b34eef06e850fc69adf606ea697aeec3b6295991681da50f7e9b95bf

data/CHANGELOG.md

@@ -1,7 +1,9 @@
-## Unreleased
+## v5.1.0 (2020-07-23)
 
   - Added a CLI method to "ping" the Solr and Source servers for a given 
     data center.
+  - Added a CLI method "errcode" to get information about the various 
+    error codes that may be returned during harvest
   - Updated the CLI harvest to return more useful error codes on failure.
 
 ## v5.0.1 (2020-07-02)

data/bin/search_solr_tools

@@ -13,7 +13,7 @@ class SolrHarvestCLI < Thor
     puts SearchSolrTools::VERSION
   end
 
-  desc 'errcode', 'Get a description of the specified error exit code.  Leave empty to see all codes'
+  desc 'errcode CODE', 'Print all exit codes bundled in CODE.  Omit CODE to print all codes'
   def errcode(code = -1)
     codes = SearchSolrTools::Errors::HarvestError.describe_exit_code(code)
 

data/lib/search_solr_tools/errors/harvest_error.rb

@@ -19,6 +19,19 @@ module SearchSolrTools
           ERRCODE_OTHER => 'General error code for non-harvest related issues'
       }.freeze
 
+      PING_ERRCODE_MAP = {
+        'ping_solr' => ERRCODE_SOLR_PING,
+        'ping_source' => ERRCODE_SOURCE_PING,
+      }
+
+      STATUS_ERRCODE_MAP = {
+          Helpers::HarvestStatus::HARVEST_NO_DOCS => ERRCODE_SOURCE_NO_RESULTS,
+          Helpers::HarvestStatus::HARVEST_FAILURE => ERRCODE_SOURCE_HARVEST_ERROR,
+          Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC => ERRCODE_DOCUMENT_INVALID,
+          Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR => ERRCODE_INGEST_ERROR,
+          Helpers::HarvestStatus::OTHER_ERROR => ERRCODE_OTHER
+      }.freeze
+
       # If code is -1, it means display all error codes
       def self.describe_exit_code(code = -1)
         code = code.to_i
@@ -52,13 +65,15 @@ module SearchSolrTools
           return ERRCODE_OTHER
         end
 
+        puts "EXIT CODE STATUS:\n#{@status_data.status}"
+
         code = 0
         code += ERRCODE_SOLR_PING unless @status_data.ping_solr
         code += ERRCODE_SOURCE_PING unless @status_data.ping_source
-        code += ERRCODE_SOURCE_NO_RESULTS if @status_data.documents_with_status(Helpers::HarvestStatus::HARVEST_NO_DOCS).size > 0
-        code += ERRCODE_SOURCE_HARVEST_ERROR if @status_data.documents_with_status(Helpers::HarvestStatus::HARVEST_FAILURE).size > 0
-        code += ERRCODE_DOCUMENT_INVALID if @status_data.documents_with_status(Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC).size > 0
-        code += ERRCODE_INGEST_ERROR if @status_data.documents_with_status(Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR).size > 0
+        code += ERRCODE_SOURCE_NO_RESULTS if @status_data.status[Helpers::HarvestStatus::HARVEST_NO_DOCS] > 0
+        code += ERRCODE_SOURCE_HARVEST_ERROR if @status_data.status[Helpers::HarvestStatus::HARVEST_FAILURE] > 0
+        code += ERRCODE_DOCUMENT_INVALID if @status_data.status[Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC] > 0
+        code += ERRCODE_INGEST_ERROR if @status_data.status[Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR] > 0
 
         code = ERRCODE_OTHER if code == 0
 

data/lib/search_solr_tools/harvesters/base.rb

@@ -59,7 +59,7 @@ module SearchSolrTools
       end
 
       # This should be overridden by child classes to implement the ability
-      # to "ping" the dataset.  Returns true if the ping is successful (or, as
+      # to "ping" the data center.  Returns true if the ping is successful (or, as
       # in this default, no ping method was defined)
       def ping_source
         puts "Harvester does not have ping method defined, assuming true"
@@ -117,8 +117,8 @@ module SearchSolrTools
         status = Helpers::HarvestStatus.new
 
         docs.each do |doc|
-          doc_status = insert_solr_doc(doc, content_type, core) #? success += 1 : failure += 1
-          status.record_document_status(doc, doc_status)
+          doc_status = insert_solr_doc(doc, content_type, core)
+          status.record_status doc_status
           doc_status == Helpers::HarvestStatus::INGEST_OK ? success += 1 : failure += 1
         end
         puts "#{success} document#{success == 1 ? '' : 's'} successfully added to Solr."

data/lib/search_solr_tools/harvesters/nsidc_json.rb

@@ -37,9 +37,10 @@ module SearchSolrTools
 
         status = insert_solr_docs result[:add_docs], Base::JSON_CONTENT_TYPE
 
-        status.record_document_status('harvest', Helpers::HarvestStatus::HARVEST_NO_DOCS) if result[:num_docs] == 0
-        status.record_multiple_document_status(result[:failure_ids],
-                                               Helpers::HarvestStatus::HARVEST_FAILURE) if result[:failure_ids].length > 0
+        status.record_status(Helpers::HarvestStatus::HARVEST_NO_DOCS) if result[:num_docs] == 0
+
+        # Record the number of harvest failures; note that if this is 0, thats OK, the status will stay at 0
+        status.record_status(Helpers::HarvestStatus::HARVEST_FAILURE, result[:failure_ids].length)
 
         raise Errors::HarvestError, status unless status.ok?
       rescue Errors::HarvestError => e

data/lib/search_solr_tools/helpers/harvest_status.rb

@@ -1,7 +1,6 @@
 module SearchSolrTools
   module Helpers
     class HarvestStatus
-
       INGEST_OK = :ok
       HARVEST_NO_DOCS = :harvest_none
       HARVEST_FAILURE = :harvest_fail
@@ -13,54 +12,33 @@ module SearchSolrTools
 
       ERROR_STATUS = [HARVEST_NO_DOCS, HARVEST_FAILURE, INGEST_ERR_INVALID_DOC, INGEST_ERR_SOLR_ERROR, OTHER_ERROR]
 
+      attr_reader :status, :ping_solr, :ping_source
+      attr_writer :ping_solr, :ping_source
+
       # init_info is an optional hash that contains the various status keys and the documents to
       # associate with them
       def initialize(init_info={})
-        @status = { INGEST_OK => [] }
+        @status = { INGEST_OK => 0 }
         @ping_solr = true
         @ping_source = true
-        ERROR_STATUS.each { |s| @status[s] = [] }
+        ERROR_STATUS.each { |s| @status[s] = 0 }
 
-        init_info.each do |key, docs|
-          @status[key] = docs if @status.include? key
+        init_info.each do |key, count|
+          @status[key] = count if @status.include? key
         end
 
         @ping_solr = init_info[PING_SOLR] if init_info.include? PING_SOLR
         @ping_source = init_info[PING_SOURCE] if init_info.include? PING_SOURCE
       end
 
-      def record_multiple_document_status(documents, doc_status)
-        documents.each { |d| record_document_status d, doc_status }
-      end
-
-      def record_document_status(document, doc_status)
-        @status[doc_status] << document
-      end
-
-      def ping_solr=(newval)
-        @ping_solr = newval
-      end
-
-      def ping_source=(newval)
-        @ping_source = newval
+      def record_status(status, count = 1)
+        @status[status] += count
       end
 
       def ok?
-        ERROR_STATUS.each { |s| return false unless @status[s].empty? }
+        ERROR_STATUS.each { |s| return false unless @status[s] == 0 }
         @ping_solr && @ping_source
       end
-
-      def ping_solr
-        @ping_solr
-      end
-
-      def ping_source
-        @ping_source
-      end
-
-      def documents_with_status(doc_status)
-        @status[doc_status]
-      end
     end
   end
 end

data/lib/search_solr_tools/version.rb

@@ -1,3 +1,3 @@
 module SearchSolrTools
-  VERSION = '5.1.0.pre.2'
+  VERSION = '5.1.0.release.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: search_solr_tools
 version: !ruby/object:Gem::Version
-  version: 5.1.0.pre.1
+  version: 5.1.0.pre.2
 platform: ruby
 authors:
 - Chris Chalstrom
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-22 00:00:00.000000000 Z
+date: 2020-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi-geos