krikri 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/models/krikri/activity.rb +20 -2
- data/lib/krikri/harvester.rb +8 -1
- data/lib/krikri/mapper.rb +20 -7
- data/lib/krikri/mapping.rb +8 -3
- data/lib/krikri/mapping_dsl/rdf_subjects.rb +3 -1
- data/lib/krikri/parser.rb +5 -3
- data/lib/krikri/version.rb +1 -1
- data/spec/lib/krikri/harvester_spec.rb +21 -0
- data/spec/lib/krikri/mapper_agent_spec.rb +33 -12
- data/spec/lib/krikri/mapper_spec.rb +44 -10
- data/spec/lib/krikri/mapping_dsl/rdf_subjects_spec.rb +17 -1
- data/spec/lib/krikri/mapping_spec.rb +31 -1
- data/spec/models/activity_spec.rb +65 -2
- data/spec/support/shared_examples/parser.rb +7 -0
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 21d519147fe5ef3a3c3d26eaf8f0c9924b8208da
|
4
|
+
data.tar.gz: 1fca577cbdf76e6a9aeb680c5659bc0db268633f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 126f351593488204a37b83749da26849b69c3e41b60571f3b473db5422349fb96c3ffec6674926882c694d54b536758aee86475540ea98a78a45432c927ac2af
|
7
|
+
data.tar.gz: c61cae684f9ae1a2b350bf008e70d7598c7407a15b411acadb7d716b5879fc96afc2a891b0689aca8ad6579f19d0310cd12f253adcdbbce9e829cd05a20e354e
|
@@ -42,12 +42,30 @@ module Krikri
|
|
42
42
|
# is passed an instance of the agent, and a URI representing this Activity.
|
43
43
|
def run
|
44
44
|
if block_given?
|
45
|
+
update_attribute(:end_time, nil) if ended?
|
45
46
|
set_start_time
|
46
|
-
|
47
|
-
|
47
|
+
begin
|
48
|
+
yield agent_instance, rdf_subject
|
49
|
+
rescue => e
|
50
|
+
Rails.logger.error("Error performing Activity: #{id}\n" \
|
51
|
+
"#{e.message}\n#{e.backtrace}")
|
52
|
+
raise e
|
53
|
+
ensure
|
54
|
+
set_end_time
|
55
|
+
end
|
48
56
|
end
|
49
57
|
end
|
50
58
|
|
59
|
+
##
|
60
|
+
# Indicates whether the activity has ended. Does not distinguish between
|
61
|
+
# successful and failed completion states.
|
62
|
+
#
|
63
|
+
# @return [Boolean] `true` if the activity has been marked as ended,
|
64
|
+
# else `false`
|
65
|
+
def ended?
|
66
|
+
!self.end_time.nil?
|
67
|
+
end
|
68
|
+
|
51
69
|
##
|
52
70
|
# Instantiates and returns an instance of the Agent class with the values in
|
53
71
|
# opts.
|
data/lib/krikri/harvester.rb
CHANGED
@@ -102,7 +102,14 @@ module Krikri
|
|
102
102
|
# @return [Boolean]
|
103
103
|
def run(activity_uri = nil)
|
104
104
|
log :info, 'harvest is running'
|
105
|
-
records.each
|
105
|
+
records.each do |rec|
|
106
|
+
begin
|
107
|
+
rec.save(activity_uri)
|
108
|
+
rescue => e
|
109
|
+
log :error, "Error harvesting record:\n#{rec.content}\n\twith message:\n#{e.message}"
|
110
|
+
next
|
111
|
+
end
|
112
|
+
end
|
106
113
|
log :info, 'harvest is done'
|
107
114
|
true
|
108
115
|
end
|
data/lib/krikri/mapper.rb
CHANGED
@@ -34,13 +34,14 @@ module Krikri
|
|
34
34
|
#
|
35
35
|
# @param name [Symbol] a unique name for the mapper in the registry.
|
36
36
|
# @param opts [Hash] options to pass to the mapping instance, options are:
|
37
|
-
# :class
|
37
|
+
# :class, :parser, and :parser_args
|
38
38
|
# @yield A block passed through to the mapping instance containing the
|
39
39
|
# mapping in the language specified by MappingDSL
|
40
40
|
def define(name, opts = {}, &block)
|
41
41
|
klass = opts.fetch(:class, DPLA::MAP::Aggregation)
|
42
42
|
parser = opts.fetch(:parser, Krikri::XmlParser)
|
43
|
-
|
43
|
+
parser_args = opts.fetch(:parser_args, nil)
|
44
|
+
map = Krikri::Mapping.new(klass, parser, *parser_args)
|
44
45
|
map.instance_eval(&block) if block_given?
|
45
46
|
Registry.register!(name, map)
|
46
47
|
end
|
@@ -57,7 +58,14 @@ module Krikri
|
|
57
58
|
# @see Mapping
|
58
59
|
def map(name, records)
|
59
60
|
records = Array(records) unless records.is_a? Enumerable
|
60
|
-
records.map
|
61
|
+
records.map do |rec|
|
62
|
+
begin
|
63
|
+
Registry.get(name).process_record(rec)
|
64
|
+
rescue => e
|
65
|
+
Rails.logger.error("Error processing mapping for #{rec.rdf_subject}" \
|
66
|
+
"\n#{e.message}\n#{e.backtrace}")
|
67
|
+
end
|
68
|
+
end
|
61
69
|
end
|
62
70
|
|
63
71
|
##
|
@@ -80,10 +88,15 @@ module Krikri
|
|
80
88
|
|
81
89
|
def run(activity_uri = nil)
|
82
90
|
Krikri::Mapper.map(name, records).each do |rec|
|
83
|
-
|
84
|
-
|
85
|
-
activity_uri
|
86
|
-
|
91
|
+
begin
|
92
|
+
rec.mint_id! if rec.node?
|
93
|
+
rec << RDF::Statement(rec, RDF::PROV.wasGeneratedBy, activity_uri) if
|
94
|
+
activity_uri
|
95
|
+
rec.save
|
96
|
+
rescue => e
|
97
|
+
Rails.logger.error("Error saving record: #{rec.rdf_subject}\n" \
|
98
|
+
"#{e.message}\n#{e.backtrace}")
|
99
|
+
end
|
87
100
|
end
|
88
101
|
end
|
89
102
|
|
data/lib/krikri/mapping.rb
CHANGED
@@ -10,14 +10,19 @@ module Krikri
|
|
10
10
|
class Mapping
|
11
11
|
include MappingDSL
|
12
12
|
|
13
|
-
attr_reader :klass, :parser
|
13
|
+
attr_reader :klass, :parser, :parser_args
|
14
14
|
|
15
15
|
##
|
16
16
|
# @param klass [Class] The model class to build in the mapping process.
|
17
17
|
# @param parser [Class] The parser class with which to process resources.
|
18
|
-
|
18
|
+
# @param parser_args [Array] The arguments to pass to the parser when
|
19
|
+
# processing records.
|
20
|
+
def initialize(klass = DPLA::MAP::Aggregation,
|
21
|
+
parser = Krikri::XmlParser,
|
22
|
+
*parser_args)
|
19
23
|
@klass = klass
|
20
24
|
@parser = parser
|
25
|
+
@parser_args = parser_args
|
21
26
|
end
|
22
27
|
|
23
28
|
##
|
@@ -27,7 +32,7 @@ module Krikri
|
|
27
32
|
def process_record(record)
|
28
33
|
mapped_record = klass.new
|
29
34
|
properties.each do |prop|
|
30
|
-
prop.to_proc.call(mapped_record, parser.parse(record))
|
35
|
+
prop.to_proc.call(mapped_record, parser.parse(record, *@parser_args))
|
31
36
|
end
|
32
37
|
mapped_record
|
33
38
|
end
|
@@ -13,7 +13,9 @@ module Krikri::MappingDSL
|
|
13
13
|
value = @value
|
14
14
|
lambda do |target, record|
|
15
15
|
value = value.call(record) if value.respond_to? :call
|
16
|
-
|
16
|
+
return target.rdf_subject if Array(value).empty?
|
17
|
+
raise "Error mapping #{record}, #{target}\t" \
|
18
|
+
"URI must be set to a single value; got #{value}" if
|
17
19
|
Array(value).count != 1
|
18
20
|
value = value.first if value.is_a? Enumerable
|
19
21
|
return target.send(setter, value) unless block
|
data/lib/krikri/parser.rb
CHANGED
@@ -20,10 +20,12 @@ module Krikri
|
|
20
20
|
# Instantiates a parser object to wrap the record. Returns the record
|
21
21
|
# as is if it is already parsed.
|
22
22
|
#
|
23
|
-
# @param record the record to parse
|
23
|
+
# @param record [Krikri::OriginalRecord, Krikri::Parser] the record to parse
|
24
|
+
# @param args [Array, nil] the arguments to pass to the parser instance,
|
25
|
+
# if any
|
24
26
|
# @return [Krikri::Parser] a parsed record object
|
25
|
-
def self.parse(record)
|
26
|
-
record.is_a?(Krikri::Parser) ? record : new(record)
|
27
|
+
def self.parse(record, *args)
|
28
|
+
record.is_a?(Krikri::Parser) ? record : new(record, *args)
|
27
29
|
end
|
28
30
|
|
29
31
|
##
|
data/lib/krikri/version.rb
CHANGED
@@ -17,4 +17,25 @@ describe Krikri::Harvester do
|
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
|
+
describe '#run' do
|
21
|
+
let(:records) { [double, double] }
|
22
|
+
|
23
|
+
before do
|
24
|
+
allow(subject).to receive(:records).and_return(records)
|
25
|
+
end
|
26
|
+
|
27
|
+
context 'when save fails' do
|
28
|
+
it 'logs error' do
|
29
|
+
records.each do |rec|
|
30
|
+
allow(rec).to receive(:save)
|
31
|
+
.and_raise(StandardError.new('my message'))
|
32
|
+
allow(rec).to receive(:content).and_return 'content'
|
33
|
+
end
|
34
|
+
message =
|
35
|
+
"Error harvesting record:\ncontent\n\twith message:\nmy message"
|
36
|
+
expect(Rails.logger).to receive(:error).with(message).twice
|
37
|
+
subject.run
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
20
41
|
end
|
@@ -20,23 +20,44 @@ describe Krikri::Mapper::Agent do
|
|
20
20
|
allow(record_double).to receive(:node?).and_return(true)
|
21
21
|
allow(record_double).to receive(:mint_id!)
|
22
22
|
allow(record_double).to receive(:save)
|
23
|
-
expect(Krikri::Mapper).to receive(:map).with(mapping_name, subject.records)
|
24
|
-
.and_return(records)
|
25
23
|
end
|
26
24
|
|
27
|
-
|
28
|
-
|
25
|
+
context 'with errors thrown' do
|
26
|
+
before do
|
27
|
+
allow(record_double).to receive(:node?).and_raise(StandardError.new)
|
28
|
+
allow(record_double).to receive(:rdf_subject).and_return('123')
|
29
|
+
allow(Krikri::Mapper).to receive(:map).and_return(records)
|
30
|
+
end
|
31
|
+
|
32
|
+
it 'logs errors' do
|
33
|
+
expect(Rails.logger).to receive(:error)
|
34
|
+
.with(start_with('Error saving record: 123'))
|
35
|
+
.exactly(3).times
|
36
|
+
subject.run(activity_uri)
|
37
|
+
end
|
29
38
|
end
|
30
39
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
40
|
+
context 'with mapped records returned' do
|
41
|
+
before do
|
42
|
+
expect(Krikri::Mapper).to receive(:map)
|
43
|
+
.with(mapping_name, subject.records)
|
44
|
+
.and_return(records)
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'calls mapper' do
|
48
|
+
subject.run
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'sets generator' do
|
52
|
+
records.each do |rec|
|
53
|
+
statement = double
|
54
|
+
allow(RDF).to receive(:Statement)
|
55
|
+
.with(rec, RDF::PROV.wasGeneratedBy, activity_uri)
|
56
|
+
.and_return(statement)
|
57
|
+
expect(rec).to receive(:<<).with(statement)
|
58
|
+
end
|
59
|
+
subject.run(activity_uri)
|
38
60
|
end
|
39
|
-
subject.run(activity_uri)
|
40
61
|
end
|
41
62
|
end
|
42
63
|
|
@@ -40,6 +40,20 @@ describe Krikri::Mapper do
|
|
40
40
|
Krikri::Mapper.define(:klass_map, class: klass)
|
41
41
|
end
|
42
42
|
|
43
|
+
it 'passes parser to mapping' do
|
44
|
+
parser = Class.new
|
45
|
+
expect(Krikri::Mapping).to receive(:new)
|
46
|
+
.with(DPLA::MAP::Aggregation, parser).once
|
47
|
+
Krikri::Mapper.define(:klass_map, parser: parser)
|
48
|
+
end
|
49
|
+
|
50
|
+
it 'passes parser_args to mapping' do
|
51
|
+
args = [1,2,3]
|
52
|
+
expect(Krikri::Mapping).to receive(:new)
|
53
|
+
.with(DPLA::MAP::Aggregation, Krikri::XmlParser, *args).once
|
54
|
+
Krikri::Mapper.define(:klass_map, parser_args: args)
|
55
|
+
end
|
56
|
+
|
43
57
|
it 'hits DSL methods' do
|
44
58
|
expect_any_instance_of(Krikri::Mapping).to receive(:dsl_method_1)
|
45
59
|
.with(:arg1, :arg2)
|
@@ -76,20 +90,40 @@ describe Krikri::Mapper do
|
|
76
90
|
end
|
77
91
|
|
78
92
|
context 'with multiple records' do
|
79
|
-
before do
|
80
|
-
records.each do |rec|
|
81
|
-
expect(mapping).to receive(:process_record).with(rec)
|
82
|
-
.and_return(:mapped_record).ordered
|
83
|
-
end
|
84
|
-
end
|
85
|
-
|
86
93
|
let(:records) do
|
87
94
|
[record.clone, record.clone, record.clone]
|
88
95
|
end
|
89
96
|
|
90
|
-
|
91
|
-
|
92
|
-
.
|
97
|
+
context 'with no errors' do
|
98
|
+
before do
|
99
|
+
records.each do |rec|
|
100
|
+
expect(mapping).to receive(:process_record).with(rec)
|
101
|
+
.and_return(:mapped_record).ordered
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
it 'returns a list of items returned by mapping' do
|
106
|
+
expect(Krikri::Mapper.map(:my_map_2, records))
|
107
|
+
.to contain_exactly(:mapped_record, :mapped_record, :mapped_record)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
context 'with errors thrown' do
|
112
|
+
before do
|
113
|
+
allow(record).to receive(:rdf_subject).and_return('123')
|
114
|
+
|
115
|
+
records.each do |rec|
|
116
|
+
allow(mapping).to receive(:process_record).with(rec)
|
117
|
+
.and_raise(StandardError.new)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
it 'logs errors and continues' do
|
122
|
+
expect(Rails.logger)
|
123
|
+
.to receive(:error).with(start_with('Error processing mapping for'))
|
124
|
+
.exactly(3).times
|
125
|
+
Krikri::Mapper.map(:my_map_2, records)
|
126
|
+
end
|
93
127
|
end
|
94
128
|
end
|
95
129
|
end
|
@@ -100,12 +100,28 @@ describe Krikri::MappingDSL::RdfSubjects do
|
|
100
100
|
let(:array_value) { [value] }
|
101
101
|
end
|
102
102
|
|
103
|
+
context 'with no values' do
|
104
|
+
let(:value) { [] }
|
105
|
+
let(:node) { RDF::Node.new }
|
106
|
+
|
107
|
+
before { allow(mapped).to receive(:rdf_subject).and_return(node) }
|
108
|
+
|
109
|
+
it 'gives the bnode subject' do
|
110
|
+
expect(subject.to_proc.call(mapped, '')).to eq node
|
111
|
+
end
|
112
|
+
|
113
|
+
it 'leaves the rdf_subject untouched' do
|
114
|
+
expect(mapped).not_to receive(:set_subject!)
|
115
|
+
subject.to_proc.call(mapped, '')
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
103
119
|
context 'with too many values' do
|
104
120
|
let(:value) { ['http://example.org/1', 'http://example.org/2'] }
|
105
121
|
|
106
122
|
it 'raises an error' do
|
107
123
|
expect { subject.to_proc.call(mapped, '') }
|
108
|
-
.to raise_error
|
124
|
+
.to raise_error
|
109
125
|
end
|
110
126
|
end
|
111
127
|
end
|
@@ -2,9 +2,22 @@ require 'spec_helper'
|
|
2
2
|
|
3
3
|
describe Krikri::Mapping do
|
4
4
|
|
5
|
-
let(:
|
5
|
+
let(:target_class) { double }
|
6
|
+
let(:parser) { double }
|
7
|
+
let(:parser_args) { [1,2,3] }
|
8
|
+
|
9
|
+
describe '#new' do
|
10
|
+
it 'accepts target class, parser, and parser arguments' do
|
11
|
+
expect(described_class.new(target_class, parser, *parser_args))
|
12
|
+
.to have_attributes(klass: target_class,
|
13
|
+
parser: parser,
|
14
|
+
parser_args: parser_args)
|
15
|
+
end
|
16
|
+
end
|
6
17
|
|
7
18
|
describe '#process_record' do
|
19
|
+
let(:record) { build(:oai_dc_record) }
|
20
|
+
|
8
21
|
it 'creates a DPLA::MAP record' do
|
9
22
|
expect(subject.process_record(record)).to be_a DPLA::MAP::Aggregation
|
10
23
|
end
|
@@ -15,6 +28,23 @@ describe Krikri::Mapping do
|
|
15
28
|
expect(new_mapping.process_record(record)).to be_a klass
|
16
29
|
end
|
17
30
|
|
31
|
+
context 'with parser' do
|
32
|
+
before do
|
33
|
+
target_instance = double
|
34
|
+
allow(target_class).to receive(:new).and_return(target_instance)
|
35
|
+
allow(target_instance).to receive(:my_property=).and_return('')
|
36
|
+
subject.my_property ''
|
37
|
+
end
|
38
|
+
|
39
|
+
subject { described_class.new(target_class, parser, *parser_args) }
|
40
|
+
|
41
|
+
it 'parses record before mapping' do
|
42
|
+
expect(parser)
|
43
|
+
.to receive(:parse).with(record, *parser_args).and_return(record)
|
44
|
+
subject.process_record(record)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
18
48
|
context 'with static properties' do
|
19
49
|
before do
|
20
50
|
subject.rightsStatement value
|
@@ -24,7 +24,7 @@ describe Krikri::Activity, type: :model do
|
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
27
|
-
describe 'start_time' do
|
27
|
+
describe '#start_time' do
|
28
28
|
before do
|
29
29
|
subject.set_start_time
|
30
30
|
end
|
@@ -34,12 +34,39 @@ describe Krikri::Activity, type: :model do
|
|
34
34
|
end
|
35
35
|
end
|
36
36
|
|
37
|
-
describe 'end_time' do
|
37
|
+
describe '#end_time' do
|
38
38
|
it 'raises an error if not started' do
|
39
39
|
expect { subject.set_end_time }.to raise_error
|
40
40
|
end
|
41
41
|
end
|
42
42
|
|
43
|
+
describe '#ended?' do
|
44
|
+
context 'before completion' do
|
45
|
+
it 'returns false' do
|
46
|
+
expect(subject).not_to be_ended
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
context 'while running' do
|
51
|
+
before { subject.set_start_time }
|
52
|
+
|
53
|
+
it 'returns false' do
|
54
|
+
expect(subject).not_to be_ended
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
context 'after completion' do
|
59
|
+
before do
|
60
|
+
subject.set_start_time
|
61
|
+
subject.set_end_time
|
62
|
+
end
|
63
|
+
|
64
|
+
it 'returns true' do
|
65
|
+
expect(subject).to be_ended
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
43
70
|
describe '#run' do
|
44
71
|
it 'runs the given block' do
|
45
72
|
expect { |b| subject.run(&b) }
|
@@ -52,6 +79,42 @@ describe Krikri::Activity, type: :model do
|
|
52
79
|
Timecop.return # come back to the present for future tests
|
53
80
|
expect(subject).to have_duration_of(duration)
|
54
81
|
end
|
82
|
+
|
83
|
+
context 'after first run' do
|
84
|
+
before do
|
85
|
+
subject.run { }
|
86
|
+
end
|
87
|
+
|
88
|
+
it 'sets end_time to nil before running' do
|
89
|
+
subject.run { expect(subject.end_time).to be_nil }
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
context 'with error' do
|
94
|
+
let(:error) { StandardError.new('my error') }
|
95
|
+
|
96
|
+
it 'logs errors' do
|
97
|
+
message = "Error performing Activity: #{subject.id}\nmy error"
|
98
|
+
expect(Rails.logger).to receive(:error).with(start_with(message))
|
99
|
+
begin
|
100
|
+
subject.run { raise error }
|
101
|
+
rescue
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
it 'rethrows error' do
|
106
|
+
expect { subject.run { raise error } }
|
107
|
+
.to raise_error StandardError
|
108
|
+
end
|
109
|
+
|
110
|
+
it 'sets end time' do
|
111
|
+
begin
|
112
|
+
subject.run { raise error }
|
113
|
+
rescue
|
114
|
+
end
|
115
|
+
expect(subject.end_time).to be_within(1.second).of(Time.now)
|
116
|
+
end
|
117
|
+
end
|
55
118
|
end
|
56
119
|
|
57
120
|
describe '#agent_instance' do
|
@@ -10,10 +10,17 @@ shared_examples_for 'a parser' do
|
|
10
10
|
end
|
11
11
|
|
12
12
|
describe '#parse' do
|
13
|
+
let(:args) { [1, 2, 3] }
|
13
14
|
it 'wraps a record in this parser' do
|
14
15
|
expect(described_class.parse(record)).to be_a described_class
|
15
16
|
end
|
16
17
|
|
18
|
+
it 'uses parser args' do
|
19
|
+
expect(described_class)
|
20
|
+
.to receive(:new).with(record, *args).and_return(:parsed)
|
21
|
+
expect(described_class.parse(record, *args)).to eq :parsed
|
22
|
+
end
|
23
|
+
|
17
24
|
it 'returns the record if already parsed' do
|
18
25
|
expect(described_class.parse(parser)).to eq parser
|
19
26
|
end
|