search_solr_tools 5.1.0.pre.1 → 5.1.0.pre.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of search_solr_tools might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6eb73f77a8b012afbb965dbeb800fd1a0fcdd3261711d2f4d218f35e0276f2c9
4
- data.tar.gz: a04a061409ad6fca267e91fd21efb6d4ee010b75024e499eeb8f451be56ba0ea
3
+ metadata.gz: 1b2767fe90ced0e512cd651820392f512e0d5ebb5bd5d1ec8d8044d01c90966f
4
+ data.tar.gz: 3c01a52136449c3281534f0353091fb607d2a28c6595392b035e14bae042c62c
5
5
  SHA512:
6
- metadata.gz: e39d629876795a63ed4802dc4cc334208300b82b42368eb7a00ecf9ad274b5af2444ac3a91204b80c50058ec1e8ecf3ea85f8aa35fc54fbd274620fef513d218
7
- data.tar.gz: 367b9a206cb10f2b2b2285c2d6882c578c4df5eab99cbf9110f942ab7e98df9509849775ffe8f23d6222182bb4f35dd4ba23121d77a7228e9efa3b1895538fac
6
+ metadata.gz: f478d8680f4ef4dd3acc5958906d65d1470f197e999f6206208a315051e3017ce54c32cc92730b78472e953b68f988bf1fea6681c2c3903a46ffe4a8bf4dbde6
7
+ data.tar.gz: 31c5f916ae86a7e2878598568e5de3caa5dade77378788a7721360bf742f8d433b8b3db7b34eef06e850fc69adf606ea697aeec3b6295991681da50f7e9b95bf
@@ -1,7 +1,9 @@
1
- ## Unreleased
1
+ ## v5.1.0 (2020-07-23)
2
2
 
3
3
  - Added a CLI method to "ping" the Solr and Source servers for a given
4
4
  data center.
5
+ - Added a CLI method "errcode" to get information about the various
6
+ error codes that may be returned during harvest
5
7
  - Updated the CLI harvest to return more useful error codes on failure.
6
8
 
7
9
  ## v5.0.1 (2020-07-02)
@@ -13,7 +13,7 @@ class SolrHarvestCLI < Thor
13
13
  puts SearchSolrTools::VERSION
14
14
  end
15
15
 
16
- desc 'errcode', 'Get a description of the specified error exit code. Leave empty to see all codes'
16
+ desc 'errcode CODE', 'Print all exit codes bundled in CODE. Omit CODE to print all codes'
17
17
  def errcode(code = -1)
18
18
  codes = SearchSolrTools::Errors::HarvestError.describe_exit_code(code)
19
19
 
@@ -19,6 +19,19 @@ module SearchSolrTools
19
19
  ERRCODE_OTHER => 'General error code for non-harvest related issues'
20
20
  }.freeze
21
21
 
22
+ PING_ERRCODE_MAP = {
23
+ 'ping_solr' => ERRCODE_SOLR_PING,
24
+ 'ping_source' => ERRCODE_SOURCE_PING,
25
+ }
26
+
27
+ STATUS_ERRCODE_MAP = {
28
+ Helpers::HarvestStatus::HARVEST_NO_DOCS => ERRCODE_SOURCE_NO_RESULTS,
29
+ Helpers::HarvestStatus::HARVEST_FAILURE => ERRCODE_SOURCE_HARVEST_ERROR,
30
+ Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC => ERRCODE_DOCUMENT_INVALID,
31
+ Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR => ERRCODE_INGEST_ERROR,
32
+ Helpers::HarvestStatus::OTHER_ERROR => ERRCODE_OTHER
33
+ }.freeze
34
+
22
35
  # If code is -1, it means display all error codes
23
36
  def self.describe_exit_code(code = -1)
24
37
  code = code.to_i
@@ -52,13 +65,15 @@ module SearchSolrTools
52
65
  return ERRCODE_OTHER
53
66
  end
54
67
 
68
+ puts "EXIT CODE STATUS:\n#{@status_data.status}"
69
+
55
70
  code = 0
56
71
  code += ERRCODE_SOLR_PING unless @status_data.ping_solr
57
72
  code += ERRCODE_SOURCE_PING unless @status_data.ping_source
58
- code += ERRCODE_SOURCE_NO_RESULTS if @status_data.documents_with_status(Helpers::HarvestStatus::HARVEST_NO_DOCS).size > 0
59
- code += ERRCODE_SOURCE_HARVEST_ERROR if @status_data.documents_with_status(Helpers::HarvestStatus::HARVEST_FAILURE).size > 0
60
- code += ERRCODE_DOCUMENT_INVALID if @status_data.documents_with_status(Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC).size > 0
61
- code += ERRCODE_INGEST_ERROR if @status_data.documents_with_status(Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR).size > 0
73
+ code += ERRCODE_SOURCE_NO_RESULTS if @status_data.status[Helpers::HarvestStatus::HARVEST_NO_DOCS] > 0
74
+ code += ERRCODE_SOURCE_HARVEST_ERROR if @status_data.status[Helpers::HarvestStatus::HARVEST_FAILURE] > 0
75
+ code += ERRCODE_DOCUMENT_INVALID if @status_data.status[Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC] > 0
76
+ code += ERRCODE_INGEST_ERROR if @status_data.status[Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR] > 0
62
77
 
63
78
  code = ERRCODE_OTHER if code == 0
64
79
 
@@ -59,7 +59,7 @@ module SearchSolrTools
59
59
  end
60
60
 
61
61
  # This should be overridden by child classes to implement the ability
62
- # to "ping" the dataset. Returns true if the ping is successful (or, as
62
+ # to "ping" the data center. Returns true if the ping is successful (or, as
63
63
  # in this default, no ping method was defined)
64
64
  def ping_source
65
65
  puts "Harvester does not have ping method defined, assuming true"
@@ -117,8 +117,8 @@ module SearchSolrTools
117
117
  status = Helpers::HarvestStatus.new
118
118
 
119
119
  docs.each do |doc|
120
- doc_status = insert_solr_doc(doc, content_type, core) #? success += 1 : failure += 1
121
- status.record_document_status(doc, doc_status)
120
+ doc_status = insert_solr_doc(doc, content_type, core)
121
+ status.record_status doc_status
122
122
  doc_status == Helpers::HarvestStatus::INGEST_OK ? success += 1 : failure += 1
123
123
  end
124
124
  puts "#{success} document#{success == 1 ? '' : 's'} successfully added to Solr."
@@ -37,9 +37,10 @@ module SearchSolrTools
37
37
 
38
38
  status = insert_solr_docs result[:add_docs], Base::JSON_CONTENT_TYPE
39
39
 
40
- status.record_document_status('harvest', Helpers::HarvestStatus::HARVEST_NO_DOCS) if result[:num_docs] == 0
41
- status.record_multiple_document_status(result[:failure_ids],
42
- Helpers::HarvestStatus::HARVEST_FAILURE) if result[:failure_ids].length > 0
40
+ status.record_status(Helpers::HarvestStatus::HARVEST_NO_DOCS) if result[:num_docs] == 0
41
+
42
+ # Record the number of harvest failures; note that if this is 0, thats OK, the status will stay at 0
43
+ status.record_status(Helpers::HarvestStatus::HARVEST_FAILURE, result[:failure_ids].length)
43
44
 
44
45
  raise Errors::HarvestError, status unless status.ok?
45
46
  rescue Errors::HarvestError => e
@@ -1,7 +1,6 @@
1
1
  module SearchSolrTools
2
2
  module Helpers
3
3
  class HarvestStatus
4
-
5
4
  INGEST_OK = :ok
6
5
  HARVEST_NO_DOCS = :harvest_none
7
6
  HARVEST_FAILURE = :harvest_fail
@@ -13,54 +12,33 @@ module SearchSolrTools
13
12
 
14
13
  ERROR_STATUS = [HARVEST_NO_DOCS, HARVEST_FAILURE, INGEST_ERR_INVALID_DOC, INGEST_ERR_SOLR_ERROR, OTHER_ERROR]
15
14
 
15
+ attr_reader :status, :ping_solr, :ping_source
16
+ attr_writer :ping_solr, :ping_source
17
+
16
18
  # init_info is an optional hash that contains the various status keys and the documents to
17
19
  # associate with them
18
20
  def initialize(init_info={})
19
- @status = { INGEST_OK => [] }
21
+ @status = { INGEST_OK => 0 }
20
22
  @ping_solr = true
21
23
  @ping_source = true
22
- ERROR_STATUS.each { |s| @status[s] = [] }
24
+ ERROR_STATUS.each { |s| @status[s] = 0 }
23
25
 
24
- init_info.each do |key, docs|
25
- @status[key] = docs if @status.include? key
26
+ init_info.each do |key, count|
27
+ @status[key] = count if @status.include? key
26
28
  end
27
29
 
28
30
  @ping_solr = init_info[PING_SOLR] if init_info.include? PING_SOLR
29
31
  @ping_source = init_info[PING_SOURCE] if init_info.include? PING_SOURCE
30
32
  end
31
33
 
32
- def record_multiple_document_status(documents, doc_status)
33
- documents.each { |d| record_document_status d, doc_status }
34
- end
35
-
36
- def record_document_status(document, doc_status)
37
- @status[doc_status] << document
38
- end
39
-
40
- def ping_solr=(newval)
41
- @ping_solr = newval
42
- end
43
-
44
- def ping_source=(newval)
45
- @ping_source = newval
34
+ def record_status(status, count = 1)
35
+ @status[status] += count
46
36
  end
47
37
 
48
38
  def ok?
49
- ERROR_STATUS.each { |s| return false unless @status[s].empty? }
39
+ ERROR_STATUS.each { |s| return false unless @status[s] == 0 }
50
40
  @ping_solr && @ping_source
51
41
  end
52
-
53
- def ping_solr
54
- @ping_solr
55
- end
56
-
57
- def ping_source
58
- @ping_source
59
- end
60
-
61
- def documents_with_status(doc_status)
62
- @status[doc_status]
63
- end
64
42
  end
65
43
  end
66
44
  end
@@ -1,3 +1,3 @@
1
1
  module SearchSolrTools
2
- VERSION = '5.1.0.pre.2'
2
+ VERSION = '5.1.0.release.1'
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: search_solr_tools
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.1.0.pre.1
4
+ version: 5.1.0.pre.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chris Chalstrom
@@ -14,7 +14,7 @@ authors:
14
14
  autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
- date: 2020-07-22 00:00:00.000000000 Z
17
+ date: 2020-07-23 00:00:00.000000000 Z
18
18
  dependencies:
19
19
  - !ruby/object:Gem::Dependency
20
20
  name: ffi-geos