zizia 2.1.0.alpha.09 → 3.0.0.alpha.01
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.rubocop.yml +5 -0
- data/app/importers/modular_importer.rb +1 -1
- data/app/jobs/zizia/start_csv_import_job.rb +1 -2
- data/docs/customizing_metadata.md +1 -3
- data/lib/zizia/hyrax_record_importer.rb +9 -12
- data/lib/zizia/importer.rb +4 -7
- data/lib/zizia/parsers/csv_parser.rb +2 -2
- data/lib/zizia/record_importer.rb +5 -18
- data/lib/zizia/spec/shared_examples/a_validator.rb +1 -18
- data/lib/zizia/spec.rb +0 -1
- data/lib/zizia/validator.rb +2 -13
- data/lib/zizia/version.rb +1 -1
- data/lib/zizia.rb +0 -13
- data/spec/integration/import_csv_spec.rb +3 -4
- data/spec/zizia/csv_format_validator_spec.rb +1 -1
- data/spec/zizia/hyrax_record_importer_spec.rb +3 -5
- data/spec/zizia/record_importer_spec.rb +4 -21
- data/spec/zizia/title_validator_spec.rb +1 -1
- data/spec/zizia_spec.rb +0 -12
- metadata +1 -7
- data/lib/zizia/log_stream.rb +0 -43
- data/lib/zizia/spec/shared_examples/a_message_stream.rb +0 -11
- data/lib/zizia/streams/formatted_message_stream.rb +0 -70
- data/spec/stdout_stream_spec.rb +0 -9
- data/spec/zizia/formatted_message_stream_spec.rb +0 -35
- data/spec/zizia/parser_spec.rb +0 -47
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 90b6fee12982a33482459065c8d486198ccacc9e98853e6a8654b57eee92334a
|
|
4
|
+
data.tar.gz: 9dc25fad947cfa77a960831b9e9a3307a6c8bbcc27adaecce762c4255a0da239
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 25e92885a6e991097b1debec7c9f2c00a01bf0ad4ff58c94e01bdf86518ba62867233d72845959a51eebe2510e0f14f6bbb7e829e685462a23e0e6f4f8f1981b
|
|
7
|
+
data.tar.gz: 36717ea07f9f8cb2e363d267afd759fcce3d7b50e2629eb6631a0895e2916387d77e174b93a48e46fc8db0641cfb93d98e5559f583a209136c5bb170993f7006
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
|
@@ -22,6 +22,7 @@ Metrics/AbcSize:
|
|
|
22
22
|
Exclude:
|
|
23
23
|
- 'spec/support/hyrax/basic_metadata.rb'
|
|
24
24
|
- 'lib/zizia/hyrax_record_importer.rb'
|
|
25
|
+
- lib/zizia/importer.rb
|
|
25
26
|
|
|
26
27
|
Metrics/BlockLength:
|
|
27
28
|
Exclude:
|
|
@@ -64,6 +65,10 @@ RSpec/ExampleLength:
|
|
|
64
65
|
- 'spec/zizia/hyrax_basic_metadata_mapper_spec.rb'
|
|
65
66
|
- 'spec/integration/import_hyrax_csv.rb'
|
|
66
67
|
|
|
68
|
+
RSpec/MessageSpies:
|
|
69
|
+
Exclude:
|
|
70
|
+
- spec/integration/import_csv_spec.rb
|
|
71
|
+
|
|
67
72
|
RSpec/MultipleExpectations:
|
|
68
73
|
Exclude:
|
|
69
74
|
- 'spec/zizia/hyrax_basic_metadata_mapper_spec.rb'
|
|
@@ -24,7 +24,7 @@ class ModularImporter
|
|
|
24
24
|
|
|
25
25
|
file = File.open(@csv_file)
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
Rails.logger.info "[zizia] event: start_import, batch_id: #{@csv_import.id}, collection_id: #{@collection_id}, user: #{@user_email}"
|
|
28
28
|
Zizia::Importer.new(parser: Zizia::CsvParser.new(file: file), record_importer: Zizia::HyraxRecordImporter.new(attributes: attrs)).import
|
|
29
29
|
file.close
|
|
30
30
|
end
|
|
@@ -6,8 +6,7 @@ module Zizia
|
|
|
6
6
|
|
|
7
7
|
def perform(csv_import_id)
|
|
8
8
|
csv_import = CsvImport.find csv_import_id
|
|
9
|
-
|
|
10
|
-
log_stream << "Starting import with batch ID: #{csv_import_id}"
|
|
9
|
+
Rails.logger.info "[zizia] Starting import with batch ID: #{csv_import_id}"
|
|
11
10
|
importer = ModularImporter.new(csv_import)
|
|
12
11
|
importer.import
|
|
13
12
|
end
|
|
@@ -36,8 +36,6 @@ It should look like this:
|
|
|
36
36
|
```ruby
|
|
37
37
|
Zizia.config do |config|
|
|
38
38
|
config.metadata_mapper_class = CustomMapper
|
|
39
|
-
config.default_info_stream = Rails.logger
|
|
40
|
-
config.default_error_stream = Rails.logger
|
|
41
39
|
end
|
|
42
40
|
```
|
|
43
|
-
This tells zizia what class to use for metadata mappings
|
|
41
|
+
This tells zizia what class to use for metadata mappings. Output is logged to `Rails.logger` and all output is prefixed with `[zizia]` for easy log parsing.
|
|
@@ -44,16 +44,13 @@ module Zizia
|
|
|
44
44
|
# batch_id: '789',
|
|
45
45
|
# deduplication_field: 'legacy_id'
|
|
46
46
|
# }
|
|
47
|
-
def initialize(
|
|
48
|
-
info_stream: Zizia.config.default_info_stream,
|
|
49
|
-
attributes: {})
|
|
47
|
+
def initialize(attributes: {})
|
|
50
48
|
self.collection_id = attributes[:collection_id]
|
|
51
49
|
self.batch_id = attributes[:batch_id]
|
|
52
50
|
self.deduplication_field = attributes[:deduplication_field]
|
|
53
51
|
set_depositor(attributes[:depositor_id])
|
|
54
52
|
@success_count = 0
|
|
55
53
|
@failure_count = 0
|
|
56
|
-
super(error_stream: error_stream, info_stream: info_stream)
|
|
57
54
|
end
|
|
58
55
|
|
|
59
56
|
# "depositor" is a required field for Hyrax. If
|
|
@@ -89,9 +86,9 @@ module Zizia
|
|
|
89
86
|
create_for(record: record) unless existing_record
|
|
90
87
|
update_for(existing_record: existing_record, update_record: record) if existing_record
|
|
91
88
|
rescue Faraday::ConnectionFailed, Ldp::HttpError => e
|
|
92
|
-
|
|
89
|
+
Rails.logger.error "[zizia] #{e}"
|
|
93
90
|
rescue RuntimeError => e
|
|
94
|
-
|
|
91
|
+
Rails.logger.error "[zizia] #{e}"
|
|
95
92
|
raise e
|
|
96
93
|
end
|
|
97
94
|
|
|
@@ -193,7 +190,7 @@ module Zizia
|
|
|
193
190
|
# We assume the object was created as expected if the actor stack returns true.
|
|
194
191
|
# Note that for now the update stack will only update metadata and update collection membership, it will not re-import files.
|
|
195
192
|
def update_for(existing_record:, update_record:)
|
|
196
|
-
|
|
193
|
+
Rails.logger.info "[zizia] event: record_update_started, batch_id: #{batch_id}, collection_id: #{collection_id}, #{deduplication_field}: #{update_record.respond_to?(deduplication_field) ? update_record.send(deduplication_field) : update_record}"
|
|
197
194
|
additional_attrs = {
|
|
198
195
|
depositor: @depositor.user_key
|
|
199
196
|
}
|
|
@@ -211,11 +208,11 @@ module Zizia
|
|
|
211
208
|
|
|
212
209
|
actor_env = Hyrax::Actors::Environment.new(existing_record, ::Ability.new(@depositor), attrs)
|
|
213
210
|
if metadata_only_middleware.update(actor_env)
|
|
214
|
-
|
|
211
|
+
Rails.logger.info "[zizia] event: record_updated, batch_id: #{batch_id}, record_id: #{existing_record.id}, collection_id: #{collection_id}, #{deduplication_field}: #{existing_record.respond_to?(deduplication_field) ? existing_record.send(deduplication_field) : existing_record}"
|
|
215
212
|
@success_count += 1
|
|
216
213
|
else
|
|
217
214
|
existing_record.errors.each do |attr, msg|
|
|
218
|
-
|
|
215
|
+
Rails.logger.error "[zizia] event: validation_failed, batch_id: #{batch_id}, collection_id: #{collection_id}, attribute: #{attr.capitalize}, message: #{msg}, record_title: record_title: #{attrs[:title] ? attrs[:title] : attrs}"
|
|
219
216
|
end
|
|
220
217
|
@failure_count += 1
|
|
221
218
|
end
|
|
@@ -224,7 +221,7 @@ module Zizia
|
|
|
224
221
|
# Create an object using the Hyrax actor stack
|
|
225
222
|
# We assume the object was created as expected if the actor stack returns true.
|
|
226
223
|
def create_for(record:)
|
|
227
|
-
|
|
224
|
+
Rails.logger.info "[zizia] event: record_import_started, batch_id: #{batch_id}, collection_id: #{collection_id}, record_title: #{record.respond_to?(:title) ? record.title : record}"
|
|
228
225
|
|
|
229
226
|
additional_attrs = {
|
|
230
227
|
uploaded_files: create_upload_files(record),
|
|
@@ -248,11 +245,11 @@ module Zizia
|
|
|
248
245
|
attrs)
|
|
249
246
|
|
|
250
247
|
if Hyrax::CurationConcern.actor.create(actor_env)
|
|
251
|
-
|
|
248
|
+
Rails.logger.info "[zizia] event: record_created, batch_id: #{batch_id}, record_id: #{created.id}, collection_id: #{collection_id}, record_title: #{attrs[:title]&.first}"
|
|
252
249
|
@success_count += 1
|
|
253
250
|
else
|
|
254
251
|
created.errors.each do |attr, msg|
|
|
255
|
-
|
|
252
|
+
Rails.logger.error "[zizia] event: validation_failed, batch_id: #{batch_id}, collection_id: #{collection_id}, attribute: #{attr.capitalize}, message: #{msg}, record_title: record_title: #{attrs[:title] ? attrs[:title] : attrs}"
|
|
256
253
|
end
|
|
257
254
|
@failure_count += 1
|
|
258
255
|
end
|
data/lib/zizia/importer.rb
CHANGED
|
@@ -31,17 +31,14 @@ module Zizia
|
|
|
31
31
|
# records.
|
|
32
32
|
# @param record_importer [RecordImporter] An object to handle import of
|
|
33
33
|
# each record
|
|
34
|
-
def initialize(parser:, record_importer: RecordImporter.new
|
|
34
|
+
def initialize(parser:, record_importer: RecordImporter.new)
|
|
35
35
|
self.parser = parser
|
|
36
36
|
self.record_importer = record_importer
|
|
37
|
-
@info_stream = info_stream
|
|
38
|
-
@error_stream = error_stream
|
|
39
37
|
end
|
|
40
38
|
|
|
41
39
|
# Do not attempt to run an import if there are no records. Instead, just write to the log.
|
|
42
40
|
def no_records_message
|
|
43
|
-
|
|
44
|
-
@error_stream << "[zizia] event: empty_import, batch_id: #{record_importer.batch_id}"
|
|
41
|
+
Rails.logger.error "[zizia] event: empty_import, batch_id: #{record_importer.batch_id}"
|
|
45
42
|
end
|
|
46
43
|
|
|
47
44
|
##
|
|
@@ -51,11 +48,11 @@ module Zizia
|
|
|
51
48
|
def import
|
|
52
49
|
no_records_message && return unless records.count.positive?
|
|
53
50
|
start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
54
|
-
|
|
51
|
+
Rails.logger.info "[zizia] event: start_import, batch_id: #{record_importer.batch_id}, expecting to import #{records.count} records."
|
|
55
52
|
records.each { |record| record_importer.import(record: record) }
|
|
56
53
|
end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
57
54
|
elapsed_time = end_time - start_time
|
|
58
|
-
|
|
55
|
+
Rails.logger.info "[zizia] event: finish_import, batch_id: #{record_importer.batch_id}, successful_record_count: #{record_importer.success_count}, failed_record_count: #{record_importer.failure_count}, elapsed_time: #{elapsed_time}, elapsed_time_per_record: #{elapsed_time / records.count}"
|
|
59
56
|
end
|
|
60
57
|
end
|
|
61
58
|
end
|
|
@@ -38,8 +38,8 @@ module Zizia
|
|
|
38
38
|
CSV.parse(file.read, headers: true).each do |row|
|
|
39
39
|
yield InputRecord.from(metadata: row)
|
|
40
40
|
end
|
|
41
|
-
rescue CSV::MalformedCSVError
|
|
42
|
-
[]
|
|
41
|
+
rescue CSV::MalformedCSVError => e
|
|
42
|
+
Rails.logger.error "[zizia] The file #{file} could not be parsed as CSV: #{e}"
|
|
43
43
|
end
|
|
44
44
|
end
|
|
45
45
|
end
|
|
@@ -3,25 +3,13 @@
|
|
|
3
3
|
module Zizia
|
|
4
4
|
class RecordImporter
|
|
5
5
|
##
|
|
6
|
-
# @!attribute [rw] error_stream
|
|
7
|
-
# @return [#<<]
|
|
8
|
-
# @!attribute [rw] info_stream
|
|
9
|
-
# @return [#<<]
|
|
10
6
|
# @!attribute [rw] batch_id
|
|
11
7
|
# @return [String] an optional batch id for this import run
|
|
12
8
|
# @!attribute [rw] success_count
|
|
13
9
|
# @return [Integer] a count of the records that were successfully created
|
|
14
10
|
# @!attribute [rw] failure_count
|
|
15
11
|
# @return [Integer] a count of the records that failed import
|
|
16
|
-
attr_accessor :
|
|
17
|
-
|
|
18
|
-
##
|
|
19
|
-
# @param error_stream [#<<]
|
|
20
|
-
def initialize(error_stream: Zizia.config.default_error_stream,
|
|
21
|
-
info_stream: Zizia.config.default_info_stream)
|
|
22
|
-
self.error_stream = error_stream
|
|
23
|
-
self.info_stream = info_stream
|
|
24
|
-
end
|
|
12
|
+
attr_accessor :batch_id, :success_count, :failure_count
|
|
25
13
|
|
|
26
14
|
##
|
|
27
15
|
# @param record [ImportRecord]
|
|
@@ -30,9 +18,9 @@ module Zizia
|
|
|
30
18
|
def import(record:)
|
|
31
19
|
create_for(record: record)
|
|
32
20
|
rescue Faraday::ConnectionFailed, Ldp::HttpError => e
|
|
33
|
-
|
|
21
|
+
Rails.logger.error "[zizia] #{e}"
|
|
34
22
|
rescue RuntimeError => e
|
|
35
|
-
|
|
23
|
+
Rails.logger.error "[zizia] #{e}"
|
|
36
24
|
raise e
|
|
37
25
|
end
|
|
38
26
|
|
|
@@ -46,12 +34,11 @@ module Zizia
|
|
|
46
34
|
private
|
|
47
35
|
|
|
48
36
|
def create_for(record:)
|
|
49
|
-
|
|
50
|
-
"#{record.respond_to?(:title) ? record.title : record}."
|
|
37
|
+
Rails.logger.info "[zizia] Creating record: #{record.respond_to?(:title) ? record.title : record}."
|
|
51
38
|
|
|
52
39
|
created = import_type.create(record.attributes)
|
|
53
40
|
|
|
54
|
-
|
|
41
|
+
Rails.logger.info "[zizia] Record created at: #{created.id}"
|
|
55
42
|
end
|
|
56
43
|
end
|
|
57
44
|
end
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
shared_examples 'a Zizia::Validator' do
|
|
4
|
-
subject(:validator) { described_class.new
|
|
5
|
-
let(:error_stream) { [] }
|
|
4
|
+
subject(:validator) { described_class.new }
|
|
6
5
|
|
|
7
6
|
define :be_a_validator_error do # |expected|
|
|
8
7
|
match { false } # { |actual| some_condition }
|
|
@@ -25,22 +24,6 @@ shared_examples 'a Zizia::Validator' do
|
|
|
25
24
|
expect(validator.validate(parser: invalid_parser)).to be_any if
|
|
26
25
|
defined?(invalid_parser)
|
|
27
26
|
end
|
|
28
|
-
|
|
29
|
-
it 'gives usable errors' do
|
|
30
|
-
pending 'we need to clarify the error type and usage'
|
|
31
|
-
|
|
32
|
-
validator.validate(parser: invalid_parser).each do |error|
|
|
33
|
-
expect(error).to be_a_validator_error
|
|
34
|
-
end
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
it 'writes errors to the error stream' do
|
|
38
|
-
if defined?(invalid_parser)
|
|
39
|
-
expect { validator.validate(parser: invalid_parser) }
|
|
40
|
-
.to change { error_stream }
|
|
41
|
-
.to include(an_instance_of(Zizia::Validator::Error))
|
|
42
|
-
end
|
|
43
|
-
end
|
|
44
27
|
end
|
|
45
28
|
end
|
|
46
29
|
end
|
data/lib/zizia/spec.rb
CHANGED
|
@@ -7,7 +7,6 @@ module Zizia
|
|
|
7
7
|
# @see https://relishapp.com/rspec/rspec-core/docs/
|
|
8
8
|
module Spec
|
|
9
9
|
require 'zizia/spec/shared_examples/a_mapper'
|
|
10
|
-
require 'zizia/spec/shared_examples/a_message_stream'
|
|
11
10
|
require 'zizia/spec/shared_examples/a_parser'
|
|
12
11
|
require 'zizia/spec/shared_examples/a_validator'
|
|
13
12
|
require 'zizia/spec/fakes/fake_parser'
|
data/lib/zizia/validator.rb
CHANGED
|
@@ -8,7 +8,7 @@ module Zizia
|
|
|
8
8
|
# validator must respond to `#validate` and return a collection of errors
|
|
9
9
|
# found during validation. If the input is valid, this collection must be
|
|
10
10
|
# empty. Otherwise, it contains any number of `Validator::Error` structs
|
|
11
|
-
# which should be
|
|
11
|
+
# which should be logged to Rails.logger by the validator.
|
|
12
12
|
#
|
|
13
13
|
# The validation process accepts an entire `Parser` and is free to inspect
|
|
14
14
|
# the input `#file` content, or view its individual `#records`.
|
|
@@ -80,24 +80,13 @@ module Zizia
|
|
|
80
80
|
end
|
|
81
81
|
end
|
|
82
82
|
|
|
83
|
-
##
|
|
84
|
-
# @!attribute [rw] error_stream
|
|
85
|
-
# @return [#<<]
|
|
86
|
-
attr_accessor :error_stream
|
|
87
|
-
|
|
88
|
-
##
|
|
89
|
-
# @param error_stream [#<<]
|
|
90
|
-
def initialize(error_stream: Zizia.config.default_error_stream)
|
|
91
|
-
self.error_stream = error_stream
|
|
92
|
-
end
|
|
93
|
-
|
|
94
83
|
##
|
|
95
84
|
# @param parser [Parser]
|
|
96
85
|
#
|
|
97
86
|
# @return [Enumerator<Error>] a collection of errors found in validation
|
|
98
87
|
def validate(parser:)
|
|
99
88
|
run_validation(parser: parser).tap do |errors|
|
|
100
|
-
errors.map { |error|
|
|
89
|
+
errors.map { |error| Rails.logger.error "[zizia] #{error}" }
|
|
101
90
|
end
|
|
102
91
|
end
|
|
103
92
|
|
data/lib/zizia/version.rb
CHANGED
data/lib/zizia.rb
CHANGED
|
@@ -18,9 +18,6 @@ require "zizia/engine"
|
|
|
18
18
|
#
|
|
19
19
|
# @example A basic configuration
|
|
20
20
|
# Zizia.config do |config|
|
|
21
|
-
# # error/info streams must respond to `#<<`
|
|
22
|
-
# config.default_error_stream = MyErrorStream.new
|
|
23
|
-
# config.default_info_stream = STDOUT
|
|
24
21
|
# end
|
|
25
22
|
#
|
|
26
23
|
module Zizia
|
|
@@ -35,7 +32,6 @@ module Zizia
|
|
|
35
32
|
end
|
|
36
33
|
module_function :config
|
|
37
34
|
|
|
38
|
-
require 'zizia/log_stream'
|
|
39
35
|
require 'zizia/version'
|
|
40
36
|
require 'zizia/metadata_mapper'
|
|
41
37
|
require 'zizia/hash_mapper'
|
|
@@ -53,18 +49,9 @@ module Zizia
|
|
|
53
49
|
##
|
|
54
50
|
# Module-wide options for `Zizia`.
|
|
55
51
|
class Configuration
|
|
56
|
-
##
|
|
57
|
-
# @!attribute [rw] default_error_stream
|
|
58
|
-
# @return [#<<]
|
|
59
|
-
# @!attribute [rw] default_info_stream
|
|
60
|
-
# @return [#<<]
|
|
61
|
-
attr_accessor :default_error_stream
|
|
62
|
-
attr_accessor :default_info_stream
|
|
63
52
|
attr_accessor :metadata_mapper_class
|
|
64
53
|
|
|
65
54
|
def initialize
|
|
66
|
-
self.default_error_stream = Zizia::LogStream.new
|
|
67
|
-
self.default_info_stream = Zizia::LogStream.new
|
|
68
55
|
self.metadata_mapper_class = Zizia::HyraxBasicMetadataMapper
|
|
69
56
|
end
|
|
70
57
|
end
|
|
@@ -18,10 +18,9 @@ describe 'importing a csv batch', :clean do
|
|
|
18
18
|
context 'with invalid CSV' do
|
|
19
19
|
let(:file) { File.open('spec/fixtures/bad_example.csv') }
|
|
20
20
|
|
|
21
|
-
it 'outputs invalid file notice to
|
|
22
|
-
expect
|
|
23
|
-
|
|
24
|
-
.to_stdout_from_any_process
|
|
21
|
+
it 'outputs invalid file notice to Rails.logger' do
|
|
22
|
+
expect(Rails.logger).to receive(:error).with("[zizia] CSV::MalformedCSVError: Illegal quoting in line 2. (Zizia::CsvFormatValidator)")
|
|
23
|
+
parser.validate
|
|
25
24
|
end
|
|
26
25
|
end
|
|
27
26
|
end
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
require 'spec_helper'
|
|
4
4
|
|
|
5
5
|
describe Zizia::CsvFormatValidator do
|
|
6
|
-
subject(:validator) { described_class.new
|
|
6
|
+
subject(:validator) { described_class.new }
|
|
7
7
|
let(:invalid_parser) { Zizia::CsvParser.new(file: invalid_file) }
|
|
8
8
|
let(:invalid_file) { File.open('spec/fixtures/bad_example.csv') }
|
|
9
9
|
|
|
@@ -3,15 +3,13 @@ require 'spec_helper'
|
|
|
3
3
|
|
|
4
4
|
describe Zizia::HyraxRecordImporter, :clean do
|
|
5
5
|
subject(:importer) do
|
|
6
|
-
described_class.new
|
|
6
|
+
described_class.new
|
|
7
7
|
end
|
|
8
8
|
|
|
9
9
|
load File.expand_path("../../support/shared_contexts/with_work_type.rb", __FILE__)
|
|
10
10
|
include_context 'with a work type'
|
|
11
11
|
|
|
12
|
-
let(:
|
|
13
|
-
let(:info_stream) { [] }
|
|
14
|
-
let(:record) { Zizia::InputRecord.from(metadata: metadata) }
|
|
12
|
+
let(:record) { Zizia::InputRecord.from(metadata: metadata) }
|
|
15
13
|
|
|
16
14
|
context 'collection id' do
|
|
17
15
|
subject(:importer) do
|
|
@@ -128,7 +126,7 @@ describe Zizia::HyraxRecordImporter, :clean do
|
|
|
128
126
|
end
|
|
129
127
|
|
|
130
128
|
context 'when depositor is passed to initializer' do
|
|
131
|
-
subject(:importer) { described_class.new(
|
|
129
|
+
subject(:importer) { described_class.new(attributes: { depositor_id: user.user_key }) }
|
|
132
130
|
|
|
133
131
|
let(:user) { ::User.new(id: '123', user_key: 'special_user@example.com') }
|
|
134
132
|
before { allow(::User).to receive(:find).and_return(user) }
|
|
@@ -4,12 +4,10 @@ require 'spec_helper'
|
|
|
4
4
|
|
|
5
5
|
describe Zizia::RecordImporter, :clean do
|
|
6
6
|
subject(:importer) do
|
|
7
|
-
described_class.new
|
|
7
|
+
described_class.new
|
|
8
8
|
end
|
|
9
9
|
|
|
10
|
-
let(:
|
|
11
|
-
let(:info_stream) { [] }
|
|
12
|
-
let(:record) { Zizia::InputRecord.from(metadata: metadata) }
|
|
10
|
+
let(:record) { Zizia::InputRecord.from(metadata: metadata) }
|
|
13
11
|
let(:metadata) do
|
|
14
12
|
{
|
|
15
13
|
'title' => 'A Title',
|
|
@@ -33,21 +31,12 @@ describe Zizia::RecordImporter, :clean do
|
|
|
33
31
|
.by 1
|
|
34
32
|
end
|
|
35
33
|
|
|
36
|
-
it 'writes to the info stream before and after create' do
|
|
37
|
-
expect { importer.import(record: record) }
|
|
38
|
-
.to change { info_stream }
|
|
39
|
-
.to contain_exactly(/^Creating record/, /^Record created/)
|
|
40
|
-
end
|
|
41
|
-
|
|
42
34
|
context 'when input record errors with LDP errors' do
|
|
43
35
|
let(:ldp_error) { Ldp::PreconditionFailed }
|
|
44
36
|
|
|
45
37
|
before { allow(record).to receive(:attributes).and_raise(ldp_error) }
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
expect { importer.import(record: record) }
|
|
49
|
-
.to change { error_stream }
|
|
50
|
-
.to contain_exactly(an_instance_of(ldp_error))
|
|
38
|
+
it 'catches the error' do
|
|
39
|
+
expect { importer.import(record: record) }.not_to raise_error(ldp_error)
|
|
51
40
|
end
|
|
52
41
|
end
|
|
53
42
|
|
|
@@ -56,12 +45,6 @@ describe Zizia::RecordImporter, :clean do
|
|
|
56
45
|
|
|
57
46
|
before { allow(record).to receive(:attributes).and_raise(custom_error) }
|
|
58
47
|
|
|
59
|
-
it 'writes errors to the error stream' do
|
|
60
|
-
expect { begin; importer.import(record: record); rescue; end }
|
|
61
|
-
.to change { error_stream }
|
|
62
|
-
.to contain_exactly(an_instance_of(custom_error))
|
|
63
|
-
end
|
|
64
|
-
|
|
65
48
|
it 'reraises error' do
|
|
66
49
|
expect { importer.import(record: record) }.to raise_error(custom_error)
|
|
67
50
|
end
|
data/spec/zizia_spec.rb
CHANGED
|
@@ -7,18 +7,6 @@ end
|
|
|
7
7
|
|
|
8
8
|
describe Zizia do
|
|
9
9
|
describe '#config' do
|
|
10
|
-
it 'can set a default error stream' do
|
|
11
|
-
expect { described_class.config { |c| c.default_error_stream = STDOUT } }
|
|
12
|
-
.to change { described_class.config.default_error_stream }
|
|
13
|
-
.to(STDOUT)
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
it 'can set a default info stream' do
|
|
17
|
-
expect { described_class.config { |c| c.default_info_stream = STDOUT } }
|
|
18
|
-
.to change { described_class.config.default_info_stream }
|
|
19
|
-
.to(STDOUT)
|
|
20
|
-
end
|
|
21
|
-
|
|
22
10
|
it 'has a default metadata mapper' do
|
|
23
11
|
expect(described_class.config.metadata_mapper_class).to eq Zizia::HyraxBasicMetadataMapper
|
|
24
12
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: zizia
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 3.0.0.alpha.01
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Data Curation Experts
|
|
@@ -329,7 +329,6 @@ files:
|
|
|
329
329
|
- lib/zizia/hyrax_record_importer.rb
|
|
330
330
|
- lib/zizia/importer.rb
|
|
331
331
|
- lib/zizia/input_record.rb
|
|
332
|
-
- lib/zizia/log_stream.rb
|
|
333
332
|
- lib/zizia/metadata_mapper.rb
|
|
334
333
|
- lib/zizia/metadata_only_stack.rb
|
|
335
334
|
- lib/zizia/parser.rb
|
|
@@ -338,10 +337,8 @@ files:
|
|
|
338
337
|
- lib/zizia/spec.rb
|
|
339
338
|
- lib/zizia/spec/fakes/fake_parser.rb
|
|
340
339
|
- lib/zizia/spec/shared_examples/a_mapper.rb
|
|
341
|
-
- lib/zizia/spec/shared_examples/a_message_stream.rb
|
|
342
340
|
- lib/zizia/spec/shared_examples/a_parser.rb
|
|
343
341
|
- lib/zizia/spec/shared_examples/a_validator.rb
|
|
344
|
-
- lib/zizia/streams/formatted_message_stream.rb
|
|
345
342
|
- lib/zizia/validator.rb
|
|
346
343
|
- lib/zizia/validators/csv_format_validator.rb
|
|
347
344
|
- lib/zizia/validators/title_validator.rb
|
|
@@ -453,7 +450,6 @@ files:
|
|
|
453
450
|
- spec/integration/import_hyrax_csv.rb
|
|
454
451
|
- spec/models/csv_import_spec.rb
|
|
455
452
|
- spec/spec_helper.rb
|
|
456
|
-
- spec/stdout_stream_spec.rb
|
|
457
453
|
- spec/support/hyrax/basic_metadata.rb
|
|
458
454
|
- spec/support/hyrax/core_metadata.rb
|
|
459
455
|
- spec/support/shared_contexts/with_work_type.rb
|
|
@@ -462,13 +458,11 @@ files:
|
|
|
462
458
|
- spec/zizia/csv_format_validator_spec.rb
|
|
463
459
|
- spec/zizia/csv_parser_spec.rb
|
|
464
460
|
- spec/zizia/csv_template_spec.rb
|
|
465
|
-
- spec/zizia/formatted_message_stream_spec.rb
|
|
466
461
|
- spec/zizia/hash_mapper_spec.rb
|
|
467
462
|
- spec/zizia/hyrax_basic_metadata_mapper_spec.rb
|
|
468
463
|
- spec/zizia/hyrax_record_importer_spec.rb
|
|
469
464
|
- spec/zizia/importer_spec.rb
|
|
470
465
|
- spec/zizia/input_record_spec.rb
|
|
471
|
-
- spec/zizia/parser_spec.rb
|
|
472
466
|
- spec/zizia/record_importer_spec.rb
|
|
473
467
|
- spec/zizia/title_validator_spec.rb
|
|
474
468
|
- spec/zizia/validator_spec.rb
|
data/lib/zizia/log_stream.rb
DELETED
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Zizia
|
|
4
|
-
class LogStream
|
|
5
|
-
##
|
|
6
|
-
# @!attribute [rw] logger
|
|
7
|
-
# @return [Logger]
|
|
8
|
-
# @!attribute [rw] severity
|
|
9
|
-
# @return [Logger::Serverity]
|
|
10
|
-
attr_accessor :logger, :severity
|
|
11
|
-
|
|
12
|
-
def initialize(logger: nil, severity: nil)
|
|
13
|
-
self.logger = logger || Logger.new(build_filename)
|
|
14
|
-
self.severity = severity || Logger::INFO
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
def <<(msg)
|
|
18
|
-
logger.add(severity, msg)
|
|
19
|
-
STDOUT << msg
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
private
|
|
23
|
-
|
|
24
|
-
def build_filename
|
|
25
|
-
return ENV['IMPORT_LOG'] if ENV['IMPORT_LOG']
|
|
26
|
-
return rails_log_name if rails_log_name
|
|
27
|
-
'./log/zizia_import.log'
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
def rails_log_name
|
|
31
|
-
case Rails.env
|
|
32
|
-
when 'production'
|
|
33
|
-
Rails.root.join('log', "csv_import.log").to_s
|
|
34
|
-
when 'development'
|
|
35
|
-
Rails.root.join('log', "dev_csv_import.log").to_s
|
|
36
|
-
when 'test'
|
|
37
|
-
Rails.root.join('log', "test_csv_import.log").to_s
|
|
38
|
-
end
|
|
39
|
-
rescue ::NameError
|
|
40
|
-
false
|
|
41
|
-
end
|
|
42
|
-
end
|
|
43
|
-
end
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Zizia
|
|
4
|
-
##
|
|
5
|
-
# A message stream that formats a message before forwarding it to an
|
|
6
|
-
# underlying {#stream} (STDOUT by default). Messages are formatted using
|
|
7
|
-
# the `#%` method; the formatter can be a string format specification like
|
|
8
|
-
# "Message received: %s".
|
|
9
|
-
#
|
|
10
|
-
# @example Using a simple formatter
|
|
11
|
-
# formatter = "Message received: %s\n"
|
|
12
|
-
# stream = Zizia::FormattedMessageStream.new(formatter: formatter)
|
|
13
|
-
#
|
|
14
|
-
# stream << "a message"
|
|
15
|
-
# # Message received: a message
|
|
16
|
-
# # => #<IO:<STDOUT>>
|
|
17
|
-
#
|
|
18
|
-
# @example A more complex formatter use case
|
|
19
|
-
# class MyFormatter
|
|
20
|
-
# def %(arg)
|
|
21
|
-
# "#{Time.now}: %s\n" % arg
|
|
22
|
-
# end
|
|
23
|
-
# end
|
|
24
|
-
#
|
|
25
|
-
# formatter = MyFormatter.new
|
|
26
|
-
# stream = Zizia::FormattedMessageStream.new(formatter: formatter)
|
|
27
|
-
#
|
|
28
|
-
# stream << 'a message'
|
|
29
|
-
# # 2018-02-02 16:10:52 -0800: a message
|
|
30
|
-
# # => #<IO:<STDOUT>>
|
|
31
|
-
#
|
|
32
|
-
# stream << 'another message'
|
|
33
|
-
# # 2018-02-02 16:10:55 -0800: another message
|
|
34
|
-
# # => #<IO:<STDOUT>>
|
|
35
|
-
#
|
|
36
|
-
class FormattedMessageStream
|
|
37
|
-
##
|
|
38
|
-
# @!attribute [rw] formatter
|
|
39
|
-
# @return [#%] A format specification
|
|
40
|
-
# @see https://ruby-doc.org/core-2.4.0/String.html#method-i-25
|
|
41
|
-
# @!attribute [rw] stream
|
|
42
|
-
# @return [#<<] an underlying stream to forward messages to after
|
|
43
|
-
# formatting
|
|
44
|
-
attr_accessor :formatter, :stream
|
|
45
|
-
|
|
46
|
-
##
|
|
47
|
-
# @param formatter [#%] A format specification
|
|
48
|
-
# @param stream [#<<] an underlying stream to forward messages to after
|
|
49
|
-
# formatting
|
|
50
|
-
#
|
|
51
|
-
# @see https://ruby-doc.org/core-2.4.0/String.html#method-i-25
|
|
52
|
-
def initialize(stream: STDOUT, formatter: "%s\n")
|
|
53
|
-
self.formatter = formatter
|
|
54
|
-
self.stream = stream
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
##
|
|
58
|
-
def <<(msg)
|
|
59
|
-
stream << format_message(msg)
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
##
|
|
63
|
-
# @param msg [#to_s]
|
|
64
|
-
#
|
|
65
|
-
# @return [String] the input, cast to a string and formatted using
|
|
66
|
-
def format_message(msg)
|
|
67
|
-
formatter % msg
|
|
68
|
-
end
|
|
69
|
-
end
|
|
70
|
-
end
|
data/spec/stdout_stream_spec.rb
DELETED
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'spec_helper'
|
|
4
|
-
require 'zizia/streams/formatted_message_stream'
|
|
5
|
-
|
|
6
|
-
describe Zizia::FormattedMessageStream do
|
|
7
|
-
subject(:stream) { described_class.new(stream: fake_stream) }
|
|
8
|
-
let(:fake_stream) { [] }
|
|
9
|
-
|
|
10
|
-
it_behaves_like 'a Zizia::MessageStream'
|
|
11
|
-
|
|
12
|
-
describe '#stream' do
|
|
13
|
-
subject(:stream) { described_class.new }
|
|
14
|
-
|
|
15
|
-
it 'is STDOUT by default' do
|
|
16
|
-
expect(stream.stream).to eq STDOUT
|
|
17
|
-
end
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
describe '#<<' do
|
|
21
|
-
it 'appends newlines by default' do
|
|
22
|
-
expect { stream << 'moomin' }
|
|
23
|
-
.to change { fake_stream }
|
|
24
|
-
.to contain_exactly("moomin\n")
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
it 'uses other % formatters' do
|
|
28
|
-
stream.formatter = "!!!%s!!!"
|
|
29
|
-
|
|
30
|
-
expect { stream << 'moomin' }
|
|
31
|
-
.to change { fake_stream }
|
|
32
|
-
.to contain_exactly('!!!moomin!!!')
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
end
|
data/spec/zizia/parser_spec.rb
DELETED
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'spec_helper'
|
|
4
|
-
|
|
5
|
-
describe Zizia::Parser do
|
|
6
|
-
subject(:parser) { described_class.new(file: file) }
|
|
7
|
-
let(:file) { :fake_file }
|
|
8
|
-
|
|
9
|
-
it_behaves_like 'a Zizia::Parser'
|
|
10
|
-
|
|
11
|
-
describe '.for' do
|
|
12
|
-
it 'raises an error' do
|
|
13
|
-
expect { described_class.for(file: file) }.to raise_error TypeError
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
context 'with a matching parser subclass' do
|
|
17
|
-
before(:context) do
|
|
18
|
-
##
|
|
19
|
-
# An importer that matches all types
|
|
20
|
-
class MyFakeParser < described_class
|
|
21
|
-
class << self
|
|
22
|
-
def match?(**_opts)
|
|
23
|
-
true
|
|
24
|
-
end
|
|
25
|
-
end
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
class NestedParser < MyFakeParser; end
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
after(:context) do
|
|
32
|
-
Object.send(:remove_const, :MyFakeParser)
|
|
33
|
-
Object.send(:remove_const, :NestedParser)
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
it 'returns an importer instance' do
|
|
37
|
-
expect(described_class.for(file: file)).to be_a NestedParser
|
|
38
|
-
end
|
|
39
|
-
end
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
describe '#records' do
|
|
43
|
-
it 'raises NotImplementedError' do
|
|
44
|
-
expect { parser.records }.to raise_error NotImplementedError
|
|
45
|
-
end
|
|
46
|
-
end
|
|
47
|
-
end
|