search_solr_tools 5.1.0.pre.1 → 5.1.0.pre.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of search_solr_tools might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -1
- data/bin/search_solr_tools +1 -1
- data/lib/search_solr_tools/errors/harvest_error.rb +19 -4
- data/lib/search_solr_tools/harvesters/base.rb +3 -3
- data/lib/search_solr_tools/harvesters/nsidc_json.rb +4 -3
- data/lib/search_solr_tools/helpers/harvest_status.rb +10 -32
- data/lib/search_solr_tools/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1b2767fe90ced0e512cd651820392f512e0d5ebb5bd5d1ec8d8044d01c90966f
|
4
|
+
data.tar.gz: 3c01a52136449c3281534f0353091fb607d2a28c6595392b035e14bae042c62c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f478d8680f4ef4dd3acc5958906d65d1470f197e999f6206208a315051e3017ce54c32cc92730b78472e953b68f988bf1fea6681c2c3903a46ffe4a8bf4dbde6
|
7
|
+
data.tar.gz: 31c5f916ae86a7e2878598568e5de3caa5dade77378788a7721360bf742f8d433b8b3db7b34eef06e850fc69adf606ea697aeec3b6295991681da50f7e9b95bf
|
data/CHANGELOG.md
CHANGED
@@ -1,7 +1,9 @@
|
|
1
|
-
##
|
1
|
+
## v5.1.0 (2020-07-23)
|
2
2
|
|
3
3
|
- Added a CLI method to "ping" the Solr and Source servers for a given
|
4
4
|
data center.
|
5
|
+
- Added a CLI method "errcode" to get information about the various
|
6
|
+
error codes that may be returned during harvest
|
5
7
|
- Updated the CLI harvest to return more useful error codes on failure.
|
6
8
|
|
7
9
|
## v5.0.1 (2020-07-02)
|
data/bin/search_solr_tools
CHANGED
@@ -13,7 +13,7 @@ class SolrHarvestCLI < Thor
|
|
13
13
|
puts SearchSolrTools::VERSION
|
14
14
|
end
|
15
15
|
|
16
|
-
desc 'errcode', '
|
16
|
+
desc 'errcode CODE', 'Print all exit codes bundled in CODE. Omit CODE to print all codes'
|
17
17
|
def errcode(code = -1)
|
18
18
|
codes = SearchSolrTools::Errors::HarvestError.describe_exit_code(code)
|
19
19
|
|
@@ -19,6 +19,19 @@ module SearchSolrTools
|
|
19
19
|
ERRCODE_OTHER => 'General error code for non-harvest related issues'
|
20
20
|
}.freeze
|
21
21
|
|
22
|
+
PING_ERRCODE_MAP = {
|
23
|
+
'ping_solr' => ERRCODE_SOLR_PING,
|
24
|
+
'ping_source' => ERRCODE_SOURCE_PING,
|
25
|
+
}
|
26
|
+
|
27
|
+
STATUS_ERRCODE_MAP = {
|
28
|
+
Helpers::HarvestStatus::HARVEST_NO_DOCS => ERRCODE_SOURCE_NO_RESULTS,
|
29
|
+
Helpers::HarvestStatus::HARVEST_FAILURE => ERRCODE_SOURCE_HARVEST_ERROR,
|
30
|
+
Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC => ERRCODE_DOCUMENT_INVALID,
|
31
|
+
Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR => ERRCODE_INGEST_ERROR,
|
32
|
+
Helpers::HarvestStatus::OTHER_ERROR => ERRCODE_OTHER
|
33
|
+
}.freeze
|
34
|
+
|
22
35
|
# If code is -1, it means display all error codes
|
23
36
|
def self.describe_exit_code(code = -1)
|
24
37
|
code = code.to_i
|
@@ -52,13 +65,15 @@ module SearchSolrTools
|
|
52
65
|
return ERRCODE_OTHER
|
53
66
|
end
|
54
67
|
|
68
|
+
puts "EXIT CODE STATUS:\n#{@status_data.status}"
|
69
|
+
|
55
70
|
code = 0
|
56
71
|
code += ERRCODE_SOLR_PING unless @status_data.ping_solr
|
57
72
|
code += ERRCODE_SOURCE_PING unless @status_data.ping_source
|
58
|
-
code += ERRCODE_SOURCE_NO_RESULTS if @status_data.
|
59
|
-
code += ERRCODE_SOURCE_HARVEST_ERROR if @status_data.
|
60
|
-
code += ERRCODE_DOCUMENT_INVALID if @status_data.
|
61
|
-
code += ERRCODE_INGEST_ERROR if @status_data.
|
73
|
+
code += ERRCODE_SOURCE_NO_RESULTS if @status_data.status[Helpers::HarvestStatus::HARVEST_NO_DOCS] > 0
|
74
|
+
code += ERRCODE_SOURCE_HARVEST_ERROR if @status_data.status[Helpers::HarvestStatus::HARVEST_FAILURE] > 0
|
75
|
+
code += ERRCODE_DOCUMENT_INVALID if @status_data.status[Helpers::HarvestStatus::INGEST_ERR_INVALID_DOC] > 0
|
76
|
+
code += ERRCODE_INGEST_ERROR if @status_data.status[Helpers::HarvestStatus::INGEST_ERR_SOLR_ERROR] > 0
|
62
77
|
|
63
78
|
code = ERRCODE_OTHER if code == 0
|
64
79
|
|
@@ -59,7 +59,7 @@ module SearchSolrTools
|
|
59
59
|
end
|
60
60
|
|
61
61
|
# This should be overridden by child classes to implement the ability
|
62
|
-
# to "ping" the
|
62
|
+
# to "ping" the data center. Returns true if the ping is successful (or, as
|
63
63
|
# in this default, no ping method was defined)
|
64
64
|
def ping_source
|
65
65
|
puts "Harvester does not have ping method defined, assuming true"
|
@@ -117,8 +117,8 @@ module SearchSolrTools
|
|
117
117
|
status = Helpers::HarvestStatus.new
|
118
118
|
|
119
119
|
docs.each do |doc|
|
120
|
-
doc_status = insert_solr_doc(doc, content_type, core)
|
121
|
-
status.
|
120
|
+
doc_status = insert_solr_doc(doc, content_type, core)
|
121
|
+
status.record_status doc_status
|
122
122
|
doc_status == Helpers::HarvestStatus::INGEST_OK ? success += 1 : failure += 1
|
123
123
|
end
|
124
124
|
puts "#{success} document#{success == 1 ? '' : 's'} successfully added to Solr."
|
@@ -37,9 +37,10 @@ module SearchSolrTools
|
|
37
37
|
|
38
38
|
status = insert_solr_docs result[:add_docs], Base::JSON_CONTENT_TYPE
|
39
39
|
|
40
|
-
status.
|
41
|
-
|
42
|
-
|
40
|
+
status.record_status(Helpers::HarvestStatus::HARVEST_NO_DOCS) if result[:num_docs] == 0
|
41
|
+
|
42
|
+
# Record the number of harvest failures; note that if this is 0, thats OK, the status will stay at 0
|
43
|
+
status.record_status(Helpers::HarvestStatus::HARVEST_FAILURE, result[:failure_ids].length)
|
43
44
|
|
44
45
|
raise Errors::HarvestError, status unless status.ok?
|
45
46
|
rescue Errors::HarvestError => e
|
@@ -1,7 +1,6 @@
|
|
1
1
|
module SearchSolrTools
|
2
2
|
module Helpers
|
3
3
|
class HarvestStatus
|
4
|
-
|
5
4
|
INGEST_OK = :ok
|
6
5
|
HARVEST_NO_DOCS = :harvest_none
|
7
6
|
HARVEST_FAILURE = :harvest_fail
|
@@ -13,54 +12,33 @@ module SearchSolrTools
|
|
13
12
|
|
14
13
|
ERROR_STATUS = [HARVEST_NO_DOCS, HARVEST_FAILURE, INGEST_ERR_INVALID_DOC, INGEST_ERR_SOLR_ERROR, OTHER_ERROR]
|
15
14
|
|
15
|
+
attr_reader :status, :ping_solr, :ping_source
|
16
|
+
attr_writer :ping_solr, :ping_source
|
17
|
+
|
16
18
|
# init_info is an optional hash that contains the various status keys and the documents to
|
17
19
|
# associate with them
|
18
20
|
def initialize(init_info={})
|
19
|
-
@status = { INGEST_OK =>
|
21
|
+
@status = { INGEST_OK => 0 }
|
20
22
|
@ping_solr = true
|
21
23
|
@ping_source = true
|
22
|
-
ERROR_STATUS.each { |s| @status[s] =
|
24
|
+
ERROR_STATUS.each { |s| @status[s] = 0 }
|
23
25
|
|
24
|
-
init_info.each do |key,
|
25
|
-
@status[key] =
|
26
|
+
init_info.each do |key, count|
|
27
|
+
@status[key] = count if @status.include? key
|
26
28
|
end
|
27
29
|
|
28
30
|
@ping_solr = init_info[PING_SOLR] if init_info.include? PING_SOLR
|
29
31
|
@ping_source = init_info[PING_SOURCE] if init_info.include? PING_SOURCE
|
30
32
|
end
|
31
33
|
|
32
|
-
def
|
33
|
-
|
34
|
-
end
|
35
|
-
|
36
|
-
def record_document_status(document, doc_status)
|
37
|
-
@status[doc_status] << document
|
38
|
-
end
|
39
|
-
|
40
|
-
def ping_solr=(newval)
|
41
|
-
@ping_solr = newval
|
42
|
-
end
|
43
|
-
|
44
|
-
def ping_source=(newval)
|
45
|
-
@ping_source = newval
|
34
|
+
def record_status(status, count = 1)
|
35
|
+
@status[status] += count
|
46
36
|
end
|
47
37
|
|
48
38
|
def ok?
|
49
|
-
ERROR_STATUS.each { |s| return false unless @status[s]
|
39
|
+
ERROR_STATUS.each { |s| return false unless @status[s] == 0 }
|
50
40
|
@ping_solr && @ping_source
|
51
41
|
end
|
52
|
-
|
53
|
-
def ping_solr
|
54
|
-
@ping_solr
|
55
|
-
end
|
56
|
-
|
57
|
-
def ping_source
|
58
|
-
@ping_source
|
59
|
-
end
|
60
|
-
|
61
|
-
def documents_with_status(doc_status)
|
62
|
-
@status[doc_status]
|
63
|
-
end
|
64
42
|
end
|
65
43
|
end
|
66
44
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: search_solr_tools
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.1.0.pre.
|
4
|
+
version: 5.1.0.pre.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Chalstrom
|
@@ -14,7 +14,7 @@ authors:
|
|
14
14
|
autorequire:
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
|
-
date: 2020-07-
|
17
|
+
date: 2020-07-23 00:00:00.000000000 Z
|
18
18
|
dependencies:
|
19
19
|
- !ruby/object:Gem::Dependency
|
20
20
|
name: ffi-geos
|