krikri 0.1.1 → 0.1.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 01b6a4cc31b96735f4e2ec59fd2b9ee25c2cc88f
4
- data.tar.gz: 06fa312088255ba9ed484c007d7bde50597b788b
3
+ metadata.gz: 21d519147fe5ef3a3c3d26eaf8f0c9924b8208da
4
+ data.tar.gz: 1fca577cbdf76e6a9aeb680c5659bc0db268633f
5
5
  SHA512:
6
- metadata.gz: 7c00770e89b37703c31c100fc589889acaf6c1fa42c1b20c340478a0c6d23b6330f016dad2b42220ab5f06b6342bb82b9bc2d698b9243de446dd6b782500522c
7
- data.tar.gz: ec6a06a36f84bd9e626013574f91a4b026af267e9e5a8c123ea16fe1574f5d97036ae472f43ca0083b81c198cf02d8aaed8c9344df3054033c2341a2a69e2058
6
+ metadata.gz: 126f351593488204a37b83749da26849b69c3e41b60571f3b473db5422349fb96c3ffec6674926882c694d54b536758aee86475540ea98a78a45432c927ac2af
7
+ data.tar.gz: c61cae684f9ae1a2b350bf008e70d7598c7407a15b411acadb7d716b5879fc96afc2a891b0689aca8ad6579f19d0310cd12f253adcdbbce9e829cd05a20e354e
@@ -42,12 +42,30 @@ module Krikri
42
42
  # is passed an instance of the agent, and a URI representing this Activity.
43
43
  def run
44
44
  if block_given?
45
+ update_attribute(:end_time, nil) if ended?
45
46
  set_start_time
46
- yield agent_instance, rdf_subject
47
- set_end_time
47
+ begin
48
+ yield agent_instance, rdf_subject
49
+ rescue => e
50
+ Rails.logger.error("Error performing Activity: #{id}\n" \
51
+ "#{e.message}\n#{e.backtrace}")
52
+ raise e
53
+ ensure
54
+ set_end_time
55
+ end
48
56
  end
49
57
  end
50
58
 
59
+ ##
60
+ # Indicates whether the activity has ended. Does not distinguish between
61
+ # successful and failed completion states.
62
+ #
63
+ # @return [Boolean] `true` if the activity has been marked as ended,
64
+ # else `false`
65
+ def ended?
66
+ !self.end_time.nil?
67
+ end
68
+
51
69
  ##
52
70
  # Instantiates and returns an instance of the Agent class with the values in
53
71
  # opts.
@@ -102,7 +102,14 @@ module Krikri
102
102
  # @return [Boolean]
103
103
  def run(activity_uri = nil)
104
104
  log :info, 'harvest is running'
105
- records.each { |rec| rec.save(activity_uri) }
105
+ records.each do |rec|
106
+ begin
107
+ rec.save(activity_uri)
108
+ rescue => e
109
+ log :error, "Error harvesting record:\n#{rec.content}\n\twith message:\n#{e.message}"
110
+ next
111
+ end
112
+ end
106
113
  log :info, 'harvest is done'
107
114
  true
108
115
  end
@@ -34,13 +34,14 @@ module Krikri
34
34
  #
35
35
  # @param name [Symbol] a unique name for the mapper in the registry.
36
36
  # @param opts [Hash] options to pass to the mapping instance, options are:
37
- # :class
37
+ # :class, :parser, and :parser_args
38
38
  # @yield A block passed through to the mapping instance containing the
39
39
  # mapping in the language specified by MappingDSL
40
40
  def define(name, opts = {}, &block)
41
41
  klass = opts.fetch(:class, DPLA::MAP::Aggregation)
42
42
  parser = opts.fetch(:parser, Krikri::XmlParser)
43
- map = Krikri::Mapping.new(klass, parser)
43
+ parser_args = opts.fetch(:parser_args, nil)
44
+ map = Krikri::Mapping.new(klass, parser, *parser_args)
44
45
  map.instance_eval(&block) if block_given?
45
46
  Registry.register!(name, map)
46
47
  end
@@ -57,7 +58,14 @@ module Krikri
57
58
  # @see Mapping
58
59
  def map(name, records)
59
60
  records = Array(records) unless records.is_a? Enumerable
60
- records.map { |rec| Registry.get(name).process_record(rec) }
61
+ records.map do |rec|
62
+ begin
63
+ Registry.get(name).process_record(rec)
64
+ rescue => e
65
+ Rails.logger.error("Error processing mapping for #{rec.rdf_subject}" \
66
+ "\n#{e.message}\n#{e.backtrace}")
67
+ end
68
+ end
61
69
  end
62
70
 
63
71
  ##
@@ -80,10 +88,15 @@ module Krikri
80
88
 
81
89
  def run(activity_uri = nil)
82
90
  Krikri::Mapper.map(name, records).each do |rec|
83
- rec.mint_id! if rec.node?
84
- rec << RDF::Statement(rec, RDF::PROV.wasGeneratedBy, activity_uri) if
85
- activity_uri
86
- rec.save
91
+ begin
92
+ rec.mint_id! if rec.node?
93
+ rec << RDF::Statement(rec, RDF::PROV.wasGeneratedBy, activity_uri) if
94
+ activity_uri
95
+ rec.save
96
+ rescue => e
97
+ Rails.logger.error("Error saving record: #{rec.rdf_subject}\n" \
98
+ "#{e.message}\n#{e.backtrace}")
99
+ end
87
100
  end
88
101
  end
89
102
 
@@ -10,14 +10,19 @@ module Krikri
10
10
  class Mapping
11
11
  include MappingDSL
12
12
 
13
- attr_reader :klass, :parser
13
+ attr_reader :klass, :parser, :parser_args
14
14
 
15
15
  ##
16
16
  # @param klass [Class] The model class to build in the mapping process.
17
17
  # @param parser [Class] The parser class with which to process resources.
18
- def initialize(klass = DPLA::MAP::Aggregation, parser = Krikri::XmlParser)
18
+ # @param parser_args [Array] The arguments to pass to the parser when
19
+ # processing records.
20
+ def initialize(klass = DPLA::MAP::Aggregation,
21
+ parser = Krikri::XmlParser,
22
+ *parser_args)
19
23
  @klass = klass
20
24
  @parser = parser
25
+ @parser_args = parser_args
21
26
  end
22
27
 
23
28
  ##
@@ -27,7 +32,7 @@ module Krikri
27
32
  def process_record(record)
28
33
  mapped_record = klass.new
29
34
  properties.each do |prop|
30
- prop.to_proc.call(mapped_record, parser.parse(record))
35
+ prop.to_proc.call(mapped_record, parser.parse(record, *@parser_args))
31
36
  end
32
37
  mapped_record
33
38
  end
@@ -13,7 +13,9 @@ module Krikri::MappingDSL
13
13
  value = @value
14
14
  lambda do |target, record|
15
15
  value = value.call(record) if value.respond_to? :call
16
- raise "URI must be set to a single value; got #{value}" if
16
+ return target.rdf_subject if Array(value).empty?
17
+ raise "Error mapping #{record}, #{target}\t" \
18
+ "URI must be set to a single value; got #{value}" if
17
19
  Array(value).count != 1
18
20
  value = value.first if value.is_a? Enumerable
19
21
  return target.send(setter, value) unless block
@@ -20,10 +20,12 @@ module Krikri
20
20
  # Instantiates a parser object to wrap the record. Returns the record
21
21
  # as is if it is already parsed.
22
22
  #
23
- # @param record the record to parse
23
+ # @param record [Krikri::OriginalRecord, Krikri::Parser] the record to parse
24
+ # @param args [Array, nil] the arguments to pass to the parser instance,
25
+ # if any
24
26
  # @return [Krikri::Parser] a parsed record object
25
- def self.parse(record)
26
- record.is_a?(Krikri::Parser) ? record : new(record)
27
+ def self.parse(record, *args)
28
+ record.is_a?(Krikri::Parser) ? record : new(record, *args)
27
29
  end
28
30
 
29
31
  ##
@@ -1,3 +1,3 @@
1
1
  module Krikri
2
- VERSION = "0.1.1"
2
+ VERSION = "0.1.2"
3
3
  end
@@ -17,4 +17,25 @@ describe Krikri::Harvester do
17
17
  end
18
18
  end
19
19
 
20
+ describe '#run' do
21
+ let(:records) { [double, double] }
22
+
23
+ before do
24
+ allow(subject).to receive(:records).and_return(records)
25
+ end
26
+
27
+ context 'when save fails' do
28
+ it 'logs error' do
29
+ records.each do |rec|
30
+ allow(rec).to receive(:save)
31
+ .and_raise(StandardError.new('my message'))
32
+ allow(rec).to receive(:content).and_return 'content'
33
+ end
34
+ message =
35
+ "Error harvesting record:\ncontent\n\twith message:\nmy message"
36
+ expect(Rails.logger).to receive(:error).with(message).twice
37
+ subject.run
38
+ end
39
+ end
40
+ end
20
41
  end
@@ -20,23 +20,44 @@ describe Krikri::Mapper::Agent do
20
20
  allow(record_double).to receive(:node?).and_return(true)
21
21
  allow(record_double).to receive(:mint_id!)
22
22
  allow(record_double).to receive(:save)
23
- expect(Krikri::Mapper).to receive(:map).with(mapping_name, subject.records)
24
- .and_return(records)
25
23
  end
26
24
 
27
- it 'calls mapper' do
28
- subject.run
25
+ context 'with errors thrown' do
26
+ before do
27
+ allow(record_double).to receive(:node?).and_raise(StandardError.new)
28
+ allow(record_double).to receive(:rdf_subject).and_return('123')
29
+ allow(Krikri::Mapper).to receive(:map).and_return(records)
30
+ end
31
+
32
+ it 'logs errors' do
33
+ expect(Rails.logger).to receive(:error)
34
+ .with(start_with('Error saving record: 123'))
35
+ .exactly(3).times
36
+ subject.run(activity_uri)
37
+ end
29
38
  end
30
39
 
31
- it 'sets generator' do
32
- records.each do |rec|
33
- statement = double
34
- allow(RDF).to receive(:Statement)
35
- .with(rec, RDF::PROV.wasGeneratedBy, activity_uri)
36
- .and_return(statement)
37
- expect(rec).to receive(:<<).with(statement)
40
+ context 'with mapped records returned' do
41
+ before do
42
+ expect(Krikri::Mapper).to receive(:map)
43
+ .with(mapping_name, subject.records)
44
+ .and_return(records)
45
+ end
46
+
47
+ it 'calls mapper' do
48
+ subject.run
49
+ end
50
+
51
+ it 'sets generator' do
52
+ records.each do |rec|
53
+ statement = double
54
+ allow(RDF).to receive(:Statement)
55
+ .with(rec, RDF::PROV.wasGeneratedBy, activity_uri)
56
+ .and_return(statement)
57
+ expect(rec).to receive(:<<).with(statement)
58
+ end
59
+ subject.run(activity_uri)
38
60
  end
39
- subject.run(activity_uri)
40
61
  end
41
62
  end
42
63
 
@@ -40,6 +40,20 @@ describe Krikri::Mapper do
40
40
  Krikri::Mapper.define(:klass_map, class: klass)
41
41
  end
42
42
 
43
+ it 'passes parser to mapping' do
44
+ parser = Class.new
45
+ expect(Krikri::Mapping).to receive(:new)
46
+ .with(DPLA::MAP::Aggregation, parser).once
47
+ Krikri::Mapper.define(:klass_map, parser: parser)
48
+ end
49
+
50
+ it 'passes parser_args to mapping' do
51
+ args = [1,2,3]
52
+ expect(Krikri::Mapping).to receive(:new)
53
+ .with(DPLA::MAP::Aggregation, Krikri::XmlParser, *args).once
54
+ Krikri::Mapper.define(:klass_map, parser_args: args)
55
+ end
56
+
43
57
  it 'hits DSL methods' do
44
58
  expect_any_instance_of(Krikri::Mapping).to receive(:dsl_method_1)
45
59
  .with(:arg1, :arg2)
@@ -76,20 +90,40 @@ describe Krikri::Mapper do
76
90
  end
77
91
 
78
92
  context 'with multiple records' do
79
- before do
80
- records.each do |rec|
81
- expect(mapping).to receive(:process_record).with(rec)
82
- .and_return(:mapped_record).ordered
83
- end
84
- end
85
-
86
93
  let(:records) do
87
94
  [record.clone, record.clone, record.clone]
88
95
  end
89
96
 
90
- it 'returns a list of items returned by mapping' do
91
- expect(Krikri::Mapper.map(:my_map_2, records))
92
- .to contain_exactly(:mapped_record, :mapped_record, :mapped_record)
97
+ context 'with no errors' do
98
+ before do
99
+ records.each do |rec|
100
+ expect(mapping).to receive(:process_record).with(rec)
101
+ .and_return(:mapped_record).ordered
102
+ end
103
+ end
104
+
105
+ it 'returns a list of items returned by mapping' do
106
+ expect(Krikri::Mapper.map(:my_map_2, records))
107
+ .to contain_exactly(:mapped_record, :mapped_record, :mapped_record)
108
+ end
109
+ end
110
+
111
+ context 'with errors thrown' do
112
+ before do
113
+ allow(record).to receive(:rdf_subject).and_return('123')
114
+
115
+ records.each do |rec|
116
+ allow(mapping).to receive(:process_record).with(rec)
117
+ .and_raise(StandardError.new)
118
+ end
119
+ end
120
+
121
+ it 'logs errors and continues' do
122
+ expect(Rails.logger)
123
+ .to receive(:error).with(start_with('Error processing mapping for'))
124
+ .exactly(3).times
125
+ Krikri::Mapper.map(:my_map_2, records)
126
+ end
93
127
  end
94
128
  end
95
129
  end
@@ -100,12 +100,28 @@ describe Krikri::MappingDSL::RdfSubjects do
100
100
  let(:array_value) { [value] }
101
101
  end
102
102
 
103
+ context 'with no values' do
104
+ let(:value) { [] }
105
+ let(:node) { RDF::Node.new }
106
+
107
+ before { allow(mapped).to receive(:rdf_subject).and_return(node) }
108
+
109
+ it 'gives the bnode subject' do
110
+ expect(subject.to_proc.call(mapped, '')).to eq node
111
+ end
112
+
113
+ it 'leaves the rdf_subject untouched' do
114
+ expect(mapped).not_to receive(:set_subject!)
115
+ subject.to_proc.call(mapped, '')
116
+ end
117
+ end
118
+
103
119
  context 'with too many values' do
104
120
  let(:value) { ['http://example.org/1', 'http://example.org/2'] }
105
121
 
106
122
  it 'raises an error' do
107
123
  expect { subject.to_proc.call(mapped, '') }
108
- .to raise_error "URI must be set to a single value; got #{value}"
124
+ .to raise_error
109
125
  end
110
126
  end
111
127
  end
@@ -2,9 +2,22 @@ require 'spec_helper'
2
2
 
3
3
  describe Krikri::Mapping do
4
4
 
5
- let(:record) { build(:oai_dc_record) }
5
+ let(:target_class) { double }
6
+ let(:parser) { double }
7
+ let(:parser_args) { [1,2,3] }
8
+
9
+ describe '#new' do
10
+ it 'accepts target class, parser, and parser arguments' do
11
+ expect(described_class.new(target_class, parser, *parser_args))
12
+ .to have_attributes(klass: target_class,
13
+ parser: parser,
14
+ parser_args: parser_args)
15
+ end
16
+ end
6
17
 
7
18
  describe '#process_record' do
19
+ let(:record) { build(:oai_dc_record) }
20
+
8
21
  it 'creates a DPLA::MAP record' do
9
22
  expect(subject.process_record(record)).to be_a DPLA::MAP::Aggregation
10
23
  end
@@ -15,6 +28,23 @@ describe Krikri::Mapping do
15
28
  expect(new_mapping.process_record(record)).to be_a klass
16
29
  end
17
30
 
31
+ context 'with parser' do
32
+ before do
33
+ target_instance = double
34
+ allow(target_class).to receive(:new).and_return(target_instance)
35
+ allow(target_instance).to receive(:my_property=).and_return('')
36
+ subject.my_property ''
37
+ end
38
+
39
+ subject { described_class.new(target_class, parser, *parser_args) }
40
+
41
+ it 'parses record before mapping' do
42
+ expect(parser)
43
+ .to receive(:parse).with(record, *parser_args).and_return(record)
44
+ subject.process_record(record)
45
+ end
46
+ end
47
+
18
48
  context 'with static properties' do
19
49
  before do
20
50
  subject.rightsStatement value
@@ -24,7 +24,7 @@ describe Krikri::Activity, type: :model do
24
24
  end
25
25
  end
26
26
 
27
- describe 'start_time' do
27
+ describe '#start_time' do
28
28
  before do
29
29
  subject.set_start_time
30
30
  end
@@ -34,12 +34,39 @@ describe Krikri::Activity, type: :model do
34
34
  end
35
35
  end
36
36
 
37
- describe 'end_time' do
37
+ describe '#end_time' do
38
38
  it 'raises an error if not started' do
39
39
  expect { subject.set_end_time }.to raise_error
40
40
  end
41
41
  end
42
42
 
43
+ describe '#ended?' do
44
+ context 'before completion' do
45
+ it 'returns false' do
46
+ expect(subject).not_to be_ended
47
+ end
48
+ end
49
+
50
+ context 'while running' do
51
+ before { subject.set_start_time }
52
+
53
+ it 'returns false' do
54
+ expect(subject).not_to be_ended
55
+ end
56
+ end
57
+
58
+ context 'after completion' do
59
+ before do
60
+ subject.set_start_time
61
+ subject.set_end_time
62
+ end
63
+
64
+ it 'returns true' do
65
+ expect(subject).to be_ended
66
+ end
67
+ end
68
+ end
69
+
43
70
  describe '#run' do
44
71
  it 'runs the given block' do
45
72
  expect { |b| subject.run(&b) }
@@ -52,6 +79,42 @@ describe Krikri::Activity, type: :model do
52
79
  Timecop.return # come back to the present for future tests
53
80
  expect(subject).to have_duration_of(duration)
54
81
  end
82
+
83
+ context 'after first run' do
84
+ before do
85
+ subject.run { }
86
+ end
87
+
88
+ it 'sets end_time to nil before running' do
89
+ subject.run { expect(subject.end_time).to be_nil }
90
+ end
91
+ end
92
+
93
+ context 'with error' do
94
+ let(:error) { StandardError.new('my error') }
95
+
96
+ it 'logs errors' do
97
+ message = "Error performing Activity: #{subject.id}\nmy error"
98
+ expect(Rails.logger).to receive(:error).with(start_with(message))
99
+ begin
100
+ subject.run { raise error }
101
+ rescue
102
+ end
103
+ end
104
+
105
+ it 'rethrows error' do
106
+ expect { subject.run { raise error } }
107
+ .to raise_error StandardError
108
+ end
109
+
110
+ it 'sets end time' do
111
+ begin
112
+ subject.run { raise error }
113
+ rescue
114
+ end
115
+ expect(subject.end_time).to be_within(1.second).of(Time.now)
116
+ end
117
+ end
55
118
  end
56
119
 
57
120
  describe '#agent_instance' do
@@ -10,10 +10,17 @@ shared_examples_for 'a parser' do
10
10
  end
11
11
 
12
12
  describe '#parse' do
13
+ let(:args) { [1, 2, 3] }
13
14
  it 'wraps a record in this parser' do
14
15
  expect(described_class.parse(record)).to be_a described_class
15
16
  end
16
17
 
18
+ it 'uses parser args' do
19
+ expect(described_class)
20
+ .to receive(:new).with(record, *args).and_return(:parsed)
21
+ expect(described_class.parse(record, *args)).to eq :parsed
22
+ end
23
+
17
24
  it 'returns the record if already parsed' do
18
25
  expect(described_class.parse(parser)).to eq parser
19
26
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: krikri
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Audrey Altman