zizia 2.1.0.alpha.09 → 3.0.0.alpha.01
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.rubocop.yml +5 -0
- data/app/importers/modular_importer.rb +1 -1
- data/app/jobs/zizia/start_csv_import_job.rb +1 -2
- data/docs/customizing_metadata.md +1 -3
- data/lib/zizia/hyrax_record_importer.rb +9 -12
- data/lib/zizia/importer.rb +4 -7
- data/lib/zizia/parsers/csv_parser.rb +2 -2
- data/lib/zizia/record_importer.rb +5 -18
- data/lib/zizia/spec/shared_examples/a_validator.rb +1 -18
- data/lib/zizia/spec.rb +0 -1
- data/lib/zizia/validator.rb +2 -13
- data/lib/zizia/version.rb +1 -1
- data/lib/zizia.rb +0 -13
- data/spec/integration/import_csv_spec.rb +3 -4
- data/spec/zizia/csv_format_validator_spec.rb +1 -1
- data/spec/zizia/hyrax_record_importer_spec.rb +3 -5
- data/spec/zizia/record_importer_spec.rb +4 -21
- data/spec/zizia/title_validator_spec.rb +1 -1
- data/spec/zizia_spec.rb +0 -12
- metadata +1 -7
- data/lib/zizia/log_stream.rb +0 -43
- data/lib/zizia/spec/shared_examples/a_message_stream.rb +0 -11
- data/lib/zizia/streams/formatted_message_stream.rb +0 -70
- data/spec/stdout_stream_spec.rb +0 -9
- data/spec/zizia/formatted_message_stream_spec.rb +0 -35
- data/spec/zizia/parser_spec.rb +0 -47
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 90b6fee12982a33482459065c8d486198ccacc9e98853e6a8654b57eee92334a
|
4
|
+
data.tar.gz: 9dc25fad947cfa77a960831b9e9a3307a6c8bbcc27adaecce762c4255a0da239
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 25e92885a6e991097b1debec7c9f2c00a01bf0ad4ff58c94e01bdf86518ba62867233d72845959a51eebe2510e0f14f6bbb7e829e685462a23e0e6f4f8f1981b
|
7
|
+
data.tar.gz: 36717ea07f9f8cb2e363d267afd759fcce3d7b50e2629eb6631a0895e2916387d77e174b93a48e46fc8db0641cfb93d98e5559f583a209136c5bb170993f7006
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
@@ -22,6 +22,7 @@ Metrics/AbcSize:
|
|
22
22
|
Exclude:
|
23
23
|
- 'spec/support/hyrax/basic_metadata.rb'
|
24
24
|
- 'lib/zizia/hyrax_record_importer.rb'
|
25
|
+
- lib/zizia/importer.rb
|
25
26
|
|
26
27
|
Metrics/BlockLength:
|
27
28
|
Exclude:
|
@@ -64,6 +65,10 @@ RSpec/ExampleLength:
|
|
64
65
|
- 'spec/zizia/hyrax_basic_metadata_mapper_spec.rb'
|
65
66
|
- 'spec/integration/import_hyrax_csv.rb'
|
66
67
|
|
68
|
+
RSpec/MessageSpies:
|
69
|
+
Exclude:
|
70
|
+
- spec/integration/import_csv_spec.rb
|
71
|
+
|
67
72
|
RSpec/MultipleExpectations:
|
68
73
|
Exclude:
|
69
74
|
- 'spec/zizia/hyrax_basic_metadata_mapper_spec.rb'
|
@@ -24,7 +24,7 @@ class ModularImporter
|
|
24
24
|
|
25
25
|
file = File.open(@csv_file)
|
26
26
|
|
27
|
-
|
27
|
+
Rails.logger.info "[zizia] event: start_import, batch_id: #{@csv_import.id}, collection_id: #{@collection_id}, user: #{@user_email}"
|
28
28
|
Zizia::Importer.new(parser: Zizia::CsvParser.new(file: file), record_importer: Zizia::HyraxRecordImporter.new(attributes: attrs)).import
|
29
29
|
file.close
|
30
30
|
end
|
@@ -6,8 +6,7 @@ module Zizia
|
|
6
6
|
|
7
7
|
def perform(csv_import_id)
|
8
8
|
csv_import = CsvImport.find csv_import_id
|
9
|
-
|
10
|
-
log_stream << "Starting import with batch ID: #{csv_import_id}"
|
9
|
+
Rails.logger.info "[zizia] Starting import with batch ID: #{csv_import_id}"
|
11
10
|
importer = ModularImporter.new(csv_import)
|
12
11
|
importer.import
|
13
12
|
end
|
@@ -36,8 +36,6 @@ It should look like this:
|
|
36
36
|
```ruby
|
37
37
|
Zizia.config do |config|
|
38
38
|
config.metadata_mapper_class = CustomMapper
|
39
|
-
config.default_info_stream = Rails.logger
|
40
|
-
config.default_error_stream = Rails.logger
|
41
39
|
end
|
42
40
|
```
|
43
|
-
This tells zizia what class to use for metadata mappings
|
41
|
+
This tells zizia what class to use for metadata mappings. Output is logged to `Rails.logger` and all output is prefixed with `[zizia]` for easy log parsing.
|
@@ -44,16 +44,13 @@ module Zizia
|
|
44
44
|
# batch_id: '789',
|
45
45
|
# deduplication_field: 'legacy_id'
|
46
46
|
# }
|
47
|
-
def initialize(
|
48
|
-
info_stream: Zizia.config.default_info_stream,
|
49
|
-
attributes: {})
|
47
|
+
def initialize(attributes: {})
|
50
48
|
self.collection_id = attributes[:collection_id]
|
51
49
|
self.batch_id = attributes[:batch_id]
|
52
50
|
self.deduplication_field = attributes[:deduplication_field]
|
53
51
|
set_depositor(attributes[:depositor_id])
|
54
52
|
@success_count = 0
|
55
53
|
@failure_count = 0
|
56
|
-
super(error_stream: error_stream, info_stream: info_stream)
|
57
54
|
end
|
58
55
|
|
59
56
|
# "depositor" is a required field for Hyrax. If
|
@@ -89,9 +86,9 @@ module Zizia
|
|
89
86
|
create_for(record: record) unless existing_record
|
90
87
|
update_for(existing_record: existing_record, update_record: record) if existing_record
|
91
88
|
rescue Faraday::ConnectionFailed, Ldp::HttpError => e
|
92
|
-
|
89
|
+
Rails.logger.error "[zizia] #{e}"
|
93
90
|
rescue RuntimeError => e
|
94
|
-
|
91
|
+
Rails.logger.error "[zizia] #{e}"
|
95
92
|
raise e
|
96
93
|
end
|
97
94
|
|
@@ -193,7 +190,7 @@ module Zizia
|
|
193
190
|
# We assume the object was created as expected if the actor stack returns true.
|
194
191
|
# Note that for now the update stack will only update metadata and update collection membership, it will not re-import files.
|
195
192
|
def update_for(existing_record:, update_record:)
|
196
|
-
|
193
|
+
Rails.logger.info "[zizia] event: record_update_started, batch_id: #{batch_id}, collection_id: #{collection_id}, #{deduplication_field}: #{update_record.respond_to?(deduplication_field) ? update_record.send(deduplication_field) : update_record}"
|
197
194
|
additional_attrs = {
|
198
195
|
depositor: @depositor.user_key
|
199
196
|
}
|
@@ -211,11 +208,11 @@ module Zizia
|
|
211
208
|
|
212
209
|
actor_env = Hyrax::Actors::Environment.new(existing_record, ::Ability.new(@depositor), attrs)
|
213
210
|
if metadata_only_middleware.update(actor_env)
|
214
|
-
|
211
|
+
Rails.logger.info "[zizia] event: record_updated, batch_id: #{batch_id}, record_id: #{existing_record.id}, collection_id: #{collection_id}, #{deduplication_field}: #{existing_record.respond_to?(deduplication_field) ? existing_record.send(deduplication_field) : existing_record}"
|
215
212
|
@success_count += 1
|
216
213
|
else
|
217
214
|
existing_record.errors.each do |attr, msg|
|
218
|
-
|
215
|
+
Rails.logger.error "[zizia] event: validation_failed, batch_id: #{batch_id}, collection_id: #{collection_id}, attribute: #{attr.capitalize}, message: #{msg}, record_title: record_title: #{attrs[:title] ? attrs[:title] : attrs}"
|
219
216
|
end
|
220
217
|
@failure_count += 1
|
221
218
|
end
|
@@ -224,7 +221,7 @@ module Zizia
|
|
224
221
|
# Create an object using the Hyrax actor stack
|
225
222
|
# We assume the object was created as expected if the actor stack returns true.
|
226
223
|
def create_for(record:)
|
227
|
-
|
224
|
+
Rails.logger.info "[zizia] event: record_import_started, batch_id: #{batch_id}, collection_id: #{collection_id}, record_title: #{record.respond_to?(:title) ? record.title : record}"
|
228
225
|
|
229
226
|
additional_attrs = {
|
230
227
|
uploaded_files: create_upload_files(record),
|
@@ -248,11 +245,11 @@ module Zizia
|
|
248
245
|
attrs)
|
249
246
|
|
250
247
|
if Hyrax::CurationConcern.actor.create(actor_env)
|
251
|
-
|
248
|
+
Rails.logger.info "[zizia] event: record_created, batch_id: #{batch_id}, record_id: #{created.id}, collection_id: #{collection_id}, record_title: #{attrs[:title]&.first}"
|
252
249
|
@success_count += 1
|
253
250
|
else
|
254
251
|
created.errors.each do |attr, msg|
|
255
|
-
|
252
|
+
Rails.logger.error "[zizia] event: validation_failed, batch_id: #{batch_id}, collection_id: #{collection_id}, attribute: #{attr.capitalize}, message: #{msg}, record_title: record_title: #{attrs[:title] ? attrs[:title] : attrs}"
|
256
253
|
end
|
257
254
|
@failure_count += 1
|
258
255
|
end
|
data/lib/zizia/importer.rb
CHANGED
@@ -31,17 +31,14 @@ module Zizia
|
|
31
31
|
# records.
|
32
32
|
# @param record_importer [RecordImporter] An object to handle import of
|
33
33
|
# each record
|
34
|
-
def initialize(parser:, record_importer: RecordImporter.new
|
34
|
+
def initialize(parser:, record_importer: RecordImporter.new)
|
35
35
|
self.parser = parser
|
36
36
|
self.record_importer = record_importer
|
37
|
-
@info_stream = info_stream
|
38
|
-
@error_stream = error_stream
|
39
37
|
end
|
40
38
|
|
41
39
|
# Do not attempt to run an import if there are no records. Instead, just write to the log.
|
42
40
|
def no_records_message
|
43
|
-
|
44
|
-
@error_stream << "[zizia] event: empty_import, batch_id: #{record_importer.batch_id}"
|
41
|
+
Rails.logger.error "[zizia] event: empty_import, batch_id: #{record_importer.batch_id}"
|
45
42
|
end
|
46
43
|
|
47
44
|
##
|
@@ -51,11 +48,11 @@ module Zizia
|
|
51
48
|
def import
|
52
49
|
no_records_message && return unless records.count.positive?
|
53
50
|
start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
54
|
-
|
51
|
+
Rails.logger.info "[zizia] event: start_import, batch_id: #{record_importer.batch_id}, expecting to import #{records.count} records."
|
55
52
|
records.each { |record| record_importer.import(record: record) }
|
56
53
|
end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
57
54
|
elapsed_time = end_time - start_time
|
58
|
-
|
55
|
+
Rails.logger.info "[zizia] event: finish_import, batch_id: #{record_importer.batch_id}, successful_record_count: #{record_importer.success_count}, failed_record_count: #{record_importer.failure_count}, elapsed_time: #{elapsed_time}, elapsed_time_per_record: #{elapsed_time / records.count}"
|
59
56
|
end
|
60
57
|
end
|
61
58
|
end
|
@@ -38,8 +38,8 @@ module Zizia
|
|
38
38
|
CSV.parse(file.read, headers: true).each do |row|
|
39
39
|
yield InputRecord.from(metadata: row)
|
40
40
|
end
|
41
|
-
rescue CSV::MalformedCSVError
|
42
|
-
[]
|
41
|
+
rescue CSV::MalformedCSVError => e
|
42
|
+
Rails.logger.error "[zizia] The file #{file} could not be parsed as CSV: #{e}"
|
43
43
|
end
|
44
44
|
end
|
45
45
|
end
|
@@ -3,25 +3,13 @@
|
|
3
3
|
module Zizia
|
4
4
|
class RecordImporter
|
5
5
|
##
|
6
|
-
# @!attribute [rw] error_stream
|
7
|
-
# @return [#<<]
|
8
|
-
# @!attribute [rw] info_stream
|
9
|
-
# @return [#<<]
|
10
6
|
# @!attribute [rw] batch_id
|
11
7
|
# @return [String] an optional batch id for this import run
|
12
8
|
# @!attribute [rw] success_count
|
13
9
|
# @return [Integer] a count of the records that were successfully created
|
14
10
|
# @!attribute [rw] failure_count
|
15
11
|
# @return [Integer] a count of the records that failed import
|
16
|
-
attr_accessor :
|
17
|
-
|
18
|
-
##
|
19
|
-
# @param error_stream [#<<]
|
20
|
-
def initialize(error_stream: Zizia.config.default_error_stream,
|
21
|
-
info_stream: Zizia.config.default_info_stream)
|
22
|
-
self.error_stream = error_stream
|
23
|
-
self.info_stream = info_stream
|
24
|
-
end
|
12
|
+
attr_accessor :batch_id, :success_count, :failure_count
|
25
13
|
|
26
14
|
##
|
27
15
|
# @param record [ImportRecord]
|
@@ -30,9 +18,9 @@ module Zizia
|
|
30
18
|
def import(record:)
|
31
19
|
create_for(record: record)
|
32
20
|
rescue Faraday::ConnectionFailed, Ldp::HttpError => e
|
33
|
-
|
21
|
+
Rails.logger.error "[zizia] #{e}"
|
34
22
|
rescue RuntimeError => e
|
35
|
-
|
23
|
+
Rails.logger.error "[zizia] #{e}"
|
36
24
|
raise e
|
37
25
|
end
|
38
26
|
|
@@ -46,12 +34,11 @@ module Zizia
|
|
46
34
|
private
|
47
35
|
|
48
36
|
def create_for(record:)
|
49
|
-
|
50
|
-
"#{record.respond_to?(:title) ? record.title : record}."
|
37
|
+
Rails.logger.info "[zizia] Creating record: #{record.respond_to?(:title) ? record.title : record}."
|
51
38
|
|
52
39
|
created = import_type.create(record.attributes)
|
53
40
|
|
54
|
-
|
41
|
+
Rails.logger.info "[zizia] Record created at: #{created.id}"
|
55
42
|
end
|
56
43
|
end
|
57
44
|
end
|
@@ -1,8 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
shared_examples 'a Zizia::Validator' do
|
4
|
-
subject(:validator) { described_class.new
|
5
|
-
let(:error_stream) { [] }
|
4
|
+
subject(:validator) { described_class.new }
|
6
5
|
|
7
6
|
define :be_a_validator_error do # |expected|
|
8
7
|
match { false } # { |actual| some_condition }
|
@@ -25,22 +24,6 @@ shared_examples 'a Zizia::Validator' do
|
|
25
24
|
expect(validator.validate(parser: invalid_parser)).to be_any if
|
26
25
|
defined?(invalid_parser)
|
27
26
|
end
|
28
|
-
|
29
|
-
it 'gives usable errors' do
|
30
|
-
pending 'we need to clarify the error type and usage'
|
31
|
-
|
32
|
-
validator.validate(parser: invalid_parser).each do |error|
|
33
|
-
expect(error).to be_a_validator_error
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
it 'writes errors to the error stream' do
|
38
|
-
if defined?(invalid_parser)
|
39
|
-
expect { validator.validate(parser: invalid_parser) }
|
40
|
-
.to change { error_stream }
|
41
|
-
.to include(an_instance_of(Zizia::Validator::Error))
|
42
|
-
end
|
43
|
-
end
|
44
27
|
end
|
45
28
|
end
|
46
29
|
end
|
data/lib/zizia/spec.rb
CHANGED
@@ -7,7 +7,6 @@ module Zizia
|
|
7
7
|
# @see https://relishapp.com/rspec/rspec-core/docs/
|
8
8
|
module Spec
|
9
9
|
require 'zizia/spec/shared_examples/a_mapper'
|
10
|
-
require 'zizia/spec/shared_examples/a_message_stream'
|
11
10
|
require 'zizia/spec/shared_examples/a_parser'
|
12
11
|
require 'zizia/spec/shared_examples/a_validator'
|
13
12
|
require 'zizia/spec/fakes/fake_parser'
|
data/lib/zizia/validator.rb
CHANGED
@@ -8,7 +8,7 @@ module Zizia
|
|
8
8
|
# validator must respond to `#validate` and return a collection of errors
|
9
9
|
# found during validation. If the input is valid, this collection must be
|
10
10
|
# empty. Otherwise, it contains any number of `Validator::Error` structs
|
11
|
-
# which should be
|
11
|
+
# which should be logged to Rails.logger by the validator.
|
12
12
|
#
|
13
13
|
# The validation process accepts an entire `Parser` and is free to inspect
|
14
14
|
# the input `#file` content, or view its individual `#records`.
|
@@ -80,24 +80,13 @@ module Zizia
|
|
80
80
|
end
|
81
81
|
end
|
82
82
|
|
83
|
-
##
|
84
|
-
# @!attribute [rw] error_stream
|
85
|
-
# @return [#<<]
|
86
|
-
attr_accessor :error_stream
|
87
|
-
|
88
|
-
##
|
89
|
-
# @param error_stream [#<<]
|
90
|
-
def initialize(error_stream: Zizia.config.default_error_stream)
|
91
|
-
self.error_stream = error_stream
|
92
|
-
end
|
93
|
-
|
94
83
|
##
|
95
84
|
# @param parser [Parser]
|
96
85
|
#
|
97
86
|
# @return [Enumerator<Error>] a collection of errors found in validation
|
98
87
|
def validate(parser:)
|
99
88
|
run_validation(parser: parser).tap do |errors|
|
100
|
-
errors.map { |error|
|
89
|
+
errors.map { |error| Rails.logger.error "[zizia] #{error}" }
|
101
90
|
end
|
102
91
|
end
|
103
92
|
|
data/lib/zizia/version.rb
CHANGED
data/lib/zizia.rb
CHANGED
@@ -18,9 +18,6 @@ require "zizia/engine"
|
|
18
18
|
#
|
19
19
|
# @example A basic configuration
|
20
20
|
# Zizia.config do |config|
|
21
|
-
# # error/info streams must respond to `#<<`
|
22
|
-
# config.default_error_stream = MyErrorStream.new
|
23
|
-
# config.default_info_stream = STDOUT
|
24
21
|
# end
|
25
22
|
#
|
26
23
|
module Zizia
|
@@ -35,7 +32,6 @@ module Zizia
|
|
35
32
|
end
|
36
33
|
module_function :config
|
37
34
|
|
38
|
-
require 'zizia/log_stream'
|
39
35
|
require 'zizia/version'
|
40
36
|
require 'zizia/metadata_mapper'
|
41
37
|
require 'zizia/hash_mapper'
|
@@ -53,18 +49,9 @@ module Zizia
|
|
53
49
|
##
|
54
50
|
# Module-wide options for `Zizia`.
|
55
51
|
class Configuration
|
56
|
-
##
|
57
|
-
# @!attribute [rw] default_error_stream
|
58
|
-
# @return [#<<]
|
59
|
-
# @!attribute [rw] default_info_stream
|
60
|
-
# @return [#<<]
|
61
|
-
attr_accessor :default_error_stream
|
62
|
-
attr_accessor :default_info_stream
|
63
52
|
attr_accessor :metadata_mapper_class
|
64
53
|
|
65
54
|
def initialize
|
66
|
-
self.default_error_stream = Zizia::LogStream.new
|
67
|
-
self.default_info_stream = Zizia::LogStream.new
|
68
55
|
self.metadata_mapper_class = Zizia::HyraxBasicMetadataMapper
|
69
56
|
end
|
70
57
|
end
|
@@ -18,10 +18,9 @@ describe 'importing a csv batch', :clean do
|
|
18
18
|
context 'with invalid CSV' do
|
19
19
|
let(:file) { File.open('spec/fixtures/bad_example.csv') }
|
20
20
|
|
21
|
-
it 'outputs invalid file notice to
|
22
|
-
expect
|
23
|
-
|
24
|
-
.to_stdout_from_any_process
|
21
|
+
it 'outputs invalid file notice to Rails.logger' do
|
22
|
+
expect(Rails.logger).to receive(:error).with("[zizia] CSV::MalformedCSVError: Illegal quoting in line 2. (Zizia::CsvFormatValidator)")
|
23
|
+
parser.validate
|
25
24
|
end
|
26
25
|
end
|
27
26
|
end
|
@@ -3,7 +3,7 @@
|
|
3
3
|
require 'spec_helper'
|
4
4
|
|
5
5
|
describe Zizia::CsvFormatValidator do
|
6
|
-
subject(:validator) { described_class.new
|
6
|
+
subject(:validator) { described_class.new }
|
7
7
|
let(:invalid_parser) { Zizia::CsvParser.new(file: invalid_file) }
|
8
8
|
let(:invalid_file) { File.open('spec/fixtures/bad_example.csv') }
|
9
9
|
|
@@ -3,15 +3,13 @@ require 'spec_helper'
|
|
3
3
|
|
4
4
|
describe Zizia::HyraxRecordImporter, :clean do
|
5
5
|
subject(:importer) do
|
6
|
-
described_class.new
|
6
|
+
described_class.new
|
7
7
|
end
|
8
8
|
|
9
9
|
load File.expand_path("../../support/shared_contexts/with_work_type.rb", __FILE__)
|
10
10
|
include_context 'with a work type'
|
11
11
|
|
12
|
-
let(:
|
13
|
-
let(:info_stream) { [] }
|
14
|
-
let(:record) { Zizia::InputRecord.from(metadata: metadata) }
|
12
|
+
let(:record) { Zizia::InputRecord.from(metadata: metadata) }
|
15
13
|
|
16
14
|
context 'collection id' do
|
17
15
|
subject(:importer) do
|
@@ -128,7 +126,7 @@ describe Zizia::HyraxRecordImporter, :clean do
|
|
128
126
|
end
|
129
127
|
|
130
128
|
context 'when depositor is passed to initializer' do
|
131
|
-
subject(:importer) { described_class.new(
|
129
|
+
subject(:importer) { described_class.new(attributes: { depositor_id: user.user_key }) }
|
132
130
|
|
133
131
|
let(:user) { ::User.new(id: '123', user_key: 'special_user@example.com') }
|
134
132
|
before { allow(::User).to receive(:find).and_return(user) }
|
@@ -4,12 +4,10 @@ require 'spec_helper'
|
|
4
4
|
|
5
5
|
describe Zizia::RecordImporter, :clean do
|
6
6
|
subject(:importer) do
|
7
|
-
described_class.new
|
7
|
+
described_class.new
|
8
8
|
end
|
9
9
|
|
10
|
-
let(:
|
11
|
-
let(:info_stream) { [] }
|
12
|
-
let(:record) { Zizia::InputRecord.from(metadata: metadata) }
|
10
|
+
let(:record) { Zizia::InputRecord.from(metadata: metadata) }
|
13
11
|
let(:metadata) do
|
14
12
|
{
|
15
13
|
'title' => 'A Title',
|
@@ -33,21 +31,12 @@ describe Zizia::RecordImporter, :clean do
|
|
33
31
|
.by 1
|
34
32
|
end
|
35
33
|
|
36
|
-
it 'writes to the info stream before and after create' do
|
37
|
-
expect { importer.import(record: record) }
|
38
|
-
.to change { info_stream }
|
39
|
-
.to contain_exactly(/^Creating record/, /^Record created/)
|
40
|
-
end
|
41
|
-
|
42
34
|
context 'when input record errors with LDP errors' do
|
43
35
|
let(:ldp_error) { Ldp::PreconditionFailed }
|
44
36
|
|
45
37
|
before { allow(record).to receive(:attributes).and_raise(ldp_error) }
|
46
|
-
|
47
|
-
|
48
|
-
expect { importer.import(record: record) }
|
49
|
-
.to change { error_stream }
|
50
|
-
.to contain_exactly(an_instance_of(ldp_error))
|
38
|
+
it 'catches the error' do
|
39
|
+
expect { importer.import(record: record) }.not_to raise_error(ldp_error)
|
51
40
|
end
|
52
41
|
end
|
53
42
|
|
@@ -56,12 +45,6 @@ describe Zizia::RecordImporter, :clean do
|
|
56
45
|
|
57
46
|
before { allow(record).to receive(:attributes).and_raise(custom_error) }
|
58
47
|
|
59
|
-
it 'writes errors to the error stream' do
|
60
|
-
expect { begin; importer.import(record: record); rescue; end }
|
61
|
-
.to change { error_stream }
|
62
|
-
.to contain_exactly(an_instance_of(custom_error))
|
63
|
-
end
|
64
|
-
|
65
48
|
it 'reraises error' do
|
66
49
|
expect { importer.import(record: record) }.to raise_error(custom_error)
|
67
50
|
end
|
data/spec/zizia_spec.rb
CHANGED
@@ -7,18 +7,6 @@ end
|
|
7
7
|
|
8
8
|
describe Zizia do
|
9
9
|
describe '#config' do
|
10
|
-
it 'can set a default error stream' do
|
11
|
-
expect { described_class.config { |c| c.default_error_stream = STDOUT } }
|
12
|
-
.to change { described_class.config.default_error_stream }
|
13
|
-
.to(STDOUT)
|
14
|
-
end
|
15
|
-
|
16
|
-
it 'can set a default info stream' do
|
17
|
-
expect { described_class.config { |c| c.default_info_stream = STDOUT } }
|
18
|
-
.to change { described_class.config.default_info_stream }
|
19
|
-
.to(STDOUT)
|
20
|
-
end
|
21
|
-
|
22
10
|
it 'has a default metadata mapper' do
|
23
11
|
expect(described_class.config.metadata_mapper_class).to eq Zizia::HyraxBasicMetadataMapper
|
24
12
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zizia
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.0.alpha.01
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Data Curation Experts
|
@@ -329,7 +329,6 @@ files:
|
|
329
329
|
- lib/zizia/hyrax_record_importer.rb
|
330
330
|
- lib/zizia/importer.rb
|
331
331
|
- lib/zizia/input_record.rb
|
332
|
-
- lib/zizia/log_stream.rb
|
333
332
|
- lib/zizia/metadata_mapper.rb
|
334
333
|
- lib/zizia/metadata_only_stack.rb
|
335
334
|
- lib/zizia/parser.rb
|
@@ -338,10 +337,8 @@ files:
|
|
338
337
|
- lib/zizia/spec.rb
|
339
338
|
- lib/zizia/spec/fakes/fake_parser.rb
|
340
339
|
- lib/zizia/spec/shared_examples/a_mapper.rb
|
341
|
-
- lib/zizia/spec/shared_examples/a_message_stream.rb
|
342
340
|
- lib/zizia/spec/shared_examples/a_parser.rb
|
343
341
|
- lib/zizia/spec/shared_examples/a_validator.rb
|
344
|
-
- lib/zizia/streams/formatted_message_stream.rb
|
345
342
|
- lib/zizia/validator.rb
|
346
343
|
- lib/zizia/validators/csv_format_validator.rb
|
347
344
|
- lib/zizia/validators/title_validator.rb
|
@@ -453,7 +450,6 @@ files:
|
|
453
450
|
- spec/integration/import_hyrax_csv.rb
|
454
451
|
- spec/models/csv_import_spec.rb
|
455
452
|
- spec/spec_helper.rb
|
456
|
-
- spec/stdout_stream_spec.rb
|
457
453
|
- spec/support/hyrax/basic_metadata.rb
|
458
454
|
- spec/support/hyrax/core_metadata.rb
|
459
455
|
- spec/support/shared_contexts/with_work_type.rb
|
@@ -462,13 +458,11 @@ files:
|
|
462
458
|
- spec/zizia/csv_format_validator_spec.rb
|
463
459
|
- spec/zizia/csv_parser_spec.rb
|
464
460
|
- spec/zizia/csv_template_spec.rb
|
465
|
-
- spec/zizia/formatted_message_stream_spec.rb
|
466
461
|
- spec/zizia/hash_mapper_spec.rb
|
467
462
|
- spec/zizia/hyrax_basic_metadata_mapper_spec.rb
|
468
463
|
- spec/zizia/hyrax_record_importer_spec.rb
|
469
464
|
- spec/zizia/importer_spec.rb
|
470
465
|
- spec/zizia/input_record_spec.rb
|
471
|
-
- spec/zizia/parser_spec.rb
|
472
466
|
- spec/zizia/record_importer_spec.rb
|
473
467
|
- spec/zizia/title_validator_spec.rb
|
474
468
|
- spec/zizia/validator_spec.rb
|
data/lib/zizia/log_stream.rb
DELETED
@@ -1,43 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Zizia
|
4
|
-
class LogStream
|
5
|
-
##
|
6
|
-
# @!attribute [rw] logger
|
7
|
-
# @return [Logger]
|
8
|
-
# @!attribute [rw] severity
|
9
|
-
# @return [Logger::Serverity]
|
10
|
-
attr_accessor :logger, :severity
|
11
|
-
|
12
|
-
def initialize(logger: nil, severity: nil)
|
13
|
-
self.logger = logger || Logger.new(build_filename)
|
14
|
-
self.severity = severity || Logger::INFO
|
15
|
-
end
|
16
|
-
|
17
|
-
def <<(msg)
|
18
|
-
logger.add(severity, msg)
|
19
|
-
STDOUT << msg
|
20
|
-
end
|
21
|
-
|
22
|
-
private
|
23
|
-
|
24
|
-
def build_filename
|
25
|
-
return ENV['IMPORT_LOG'] if ENV['IMPORT_LOG']
|
26
|
-
return rails_log_name if rails_log_name
|
27
|
-
'./log/zizia_import.log'
|
28
|
-
end
|
29
|
-
|
30
|
-
def rails_log_name
|
31
|
-
case Rails.env
|
32
|
-
when 'production'
|
33
|
-
Rails.root.join('log', "csv_import.log").to_s
|
34
|
-
when 'development'
|
35
|
-
Rails.root.join('log', "dev_csv_import.log").to_s
|
36
|
-
when 'test'
|
37
|
-
Rails.root.join('log', "test_csv_import.log").to_s
|
38
|
-
end
|
39
|
-
rescue ::NameError
|
40
|
-
false
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
@@ -1,70 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Zizia
|
4
|
-
##
|
5
|
-
# A message stream that formats a message before forwarding it to an
|
6
|
-
# underlying {#stream} (STDOUT by default). Messages are formatted using
|
7
|
-
# the `#%` method; the formatter can be a string format specification like
|
8
|
-
# "Message received: %s".
|
9
|
-
#
|
10
|
-
# @example Using a simple formatter
|
11
|
-
# formatter = "Message received: %s\n"
|
12
|
-
# stream = Zizia::FormattedMessageStream.new(formatter: formatter)
|
13
|
-
#
|
14
|
-
# stream << "a message"
|
15
|
-
# # Message received: a message
|
16
|
-
# # => #<IO:<STDOUT>>
|
17
|
-
#
|
18
|
-
# @example A more complex formatter use case
|
19
|
-
# class MyFormatter
|
20
|
-
# def %(arg)
|
21
|
-
# "#{Time.now}: %s\n" % arg
|
22
|
-
# end
|
23
|
-
# end
|
24
|
-
#
|
25
|
-
# formatter = MyFormatter.new
|
26
|
-
# stream = Zizia::FormattedMessageStream.new(formatter: formatter)
|
27
|
-
#
|
28
|
-
# stream << 'a message'
|
29
|
-
# # 2018-02-02 16:10:52 -0800: a message
|
30
|
-
# # => #<IO:<STDOUT>>
|
31
|
-
#
|
32
|
-
# stream << 'another message'
|
33
|
-
# # 2018-02-02 16:10:55 -0800: another message
|
34
|
-
# # => #<IO:<STDOUT>>
|
35
|
-
#
|
36
|
-
class FormattedMessageStream
|
37
|
-
##
|
38
|
-
# @!attribute [rw] formatter
|
39
|
-
# @return [#%] A format specification
|
40
|
-
# @see https://ruby-doc.org/core-2.4.0/String.html#method-i-25
|
41
|
-
# @!attribute [rw] stream
|
42
|
-
# @return [#<<] an underlying stream to forward messages to after
|
43
|
-
# formatting
|
44
|
-
attr_accessor :formatter, :stream
|
45
|
-
|
46
|
-
##
|
47
|
-
# @param formatter [#%] A format specification
|
48
|
-
# @param stream [#<<] an underlying stream to forward messages to after
|
49
|
-
# formatting
|
50
|
-
#
|
51
|
-
# @see https://ruby-doc.org/core-2.4.0/String.html#method-i-25
|
52
|
-
def initialize(stream: STDOUT, formatter: "%s\n")
|
53
|
-
self.formatter = formatter
|
54
|
-
self.stream = stream
|
55
|
-
end
|
56
|
-
|
57
|
-
##
|
58
|
-
def <<(msg)
|
59
|
-
stream << format_message(msg)
|
60
|
-
end
|
61
|
-
|
62
|
-
##
|
63
|
-
# @param msg [#to_s]
|
64
|
-
#
|
65
|
-
# @return [String] the input, cast to a string and formatted using
|
66
|
-
def format_message(msg)
|
67
|
-
formatter % msg
|
68
|
-
end
|
69
|
-
end
|
70
|
-
end
|
data/spec/stdout_stream_spec.rb
DELETED
@@ -1,35 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'spec_helper'
|
4
|
-
require 'zizia/streams/formatted_message_stream'
|
5
|
-
|
6
|
-
describe Zizia::FormattedMessageStream do
|
7
|
-
subject(:stream) { described_class.new(stream: fake_stream) }
|
8
|
-
let(:fake_stream) { [] }
|
9
|
-
|
10
|
-
it_behaves_like 'a Zizia::MessageStream'
|
11
|
-
|
12
|
-
describe '#stream' do
|
13
|
-
subject(:stream) { described_class.new }
|
14
|
-
|
15
|
-
it 'is STDOUT by default' do
|
16
|
-
expect(stream.stream).to eq STDOUT
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
describe '#<<' do
|
21
|
-
it 'appends newlines by default' do
|
22
|
-
expect { stream << 'moomin' }
|
23
|
-
.to change { fake_stream }
|
24
|
-
.to contain_exactly("moomin\n")
|
25
|
-
end
|
26
|
-
|
27
|
-
it 'uses other % formatters' do
|
28
|
-
stream.formatter = "!!!%s!!!"
|
29
|
-
|
30
|
-
expect { stream << 'moomin' }
|
31
|
-
.to change { fake_stream }
|
32
|
-
.to contain_exactly('!!!moomin!!!')
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
data/spec/zizia/parser_spec.rb
DELETED
@@ -1,47 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'spec_helper'
|
4
|
-
|
5
|
-
describe Zizia::Parser do
|
6
|
-
subject(:parser) { described_class.new(file: file) }
|
7
|
-
let(:file) { :fake_file }
|
8
|
-
|
9
|
-
it_behaves_like 'a Zizia::Parser'
|
10
|
-
|
11
|
-
describe '.for' do
|
12
|
-
it 'raises an error' do
|
13
|
-
expect { described_class.for(file: file) }.to raise_error TypeError
|
14
|
-
end
|
15
|
-
|
16
|
-
context 'with a matching parser subclass' do
|
17
|
-
before(:context) do
|
18
|
-
##
|
19
|
-
# An importer that matches all types
|
20
|
-
class MyFakeParser < described_class
|
21
|
-
class << self
|
22
|
-
def match?(**_opts)
|
23
|
-
true
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
class NestedParser < MyFakeParser; end
|
29
|
-
end
|
30
|
-
|
31
|
-
after(:context) do
|
32
|
-
Object.send(:remove_const, :MyFakeParser)
|
33
|
-
Object.send(:remove_const, :NestedParser)
|
34
|
-
end
|
35
|
-
|
36
|
-
it 'returns an importer instance' do
|
37
|
-
expect(described_class.for(file: file)).to be_a NestedParser
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
describe '#records' do
|
43
|
-
it 'raises NotImplementedError' do
|
44
|
-
expect { parser.records }.to raise_error NotImplementedError
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|