darlingtonia 0.1.1 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +4 -0
- data/CHANGELOG.md +19 -0
- data/README.md +21 -1
- data/lib/darlingtonia.rb +42 -4
- data/lib/darlingtonia/hash_mapper.rb +20 -21
- data/lib/darlingtonia/metadata_mapper.rb +84 -0
- data/lib/darlingtonia/parser.rb +33 -6
- data/lib/darlingtonia/parsers/csv_parser.rb +9 -1
- data/lib/darlingtonia/record_importer.rb +32 -1
- data/lib/darlingtonia/validator.rb +67 -7
- data/lib/darlingtonia/validators/csv_format_validator.rb +3 -1
- data/lib/darlingtonia/validators/title_validator.rb +30 -0
- data/lib/darlingtonia/version.rb +1 -1
- data/spec/darlingtonia/csv_format_validator_spec.rb +1 -1
- data/spec/darlingtonia/csv_parser_spec.rb +14 -0
- data/spec/darlingtonia/parser_spec.rb +3 -3
- data/spec/darlingtonia/record_importer_spec.rb +62 -0
- data/spec/darlingtonia/title_validator_spec.rb +23 -0
- data/spec/darlingtonia/validator_spec.rb +3 -1
- data/spec/darlingtonia_spec.rb +25 -0
- data/spec/integration/import_csv_spec.rb +13 -24
- data/spec/spec_helper.rb +3 -0
- data/spec/support/shared_contexts/with_work_type.rb +28 -0
- data/spec/support/shared_examples/a_validator.rb +12 -3
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4d2846b9bcfda0f77bed4980b06ffebeea86fa2d
|
4
|
+
data.tar.gz: 5768f600be72e578ef9969e84d1d71d3e5160c06
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1565987780f909a9ba63b872c68da62a6f782c46d5363930b188a7c64e63bc323e1cdbfb1d54274291b6a6dfea046a6a87d672648d7a11d6a707dafb25e11db8
|
7
|
+
data.tar.gz: ddae5b051da31ee14317e09b5c36597cd3ec6ad550d6ddd1585ac4d30ba77dccae2cbbbcf0deb8cf6d790bf6a37be87c46f9a766502423603e8609b27b1beb00
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,22 @@
|
|
1
|
+
0.2.0 - Wed Jan 17, 2018
|
2
|
+
------------------------
|
3
|
+
|
4
|
+
Error & info streams.
|
5
|
+
|
6
|
+
- Extend `Parser` subclasses to define `DEFAULT_VALIDATORS` to hard code
|
7
|
+
a default validator list.
|
8
|
+
- Support streaming errors from `Validator` to an error stream (`#<<`).
|
9
|
+
- Add configuration at `Darlingtonia.config` to set `default_error_stream`.
|
10
|
+
- Introduce `MetadataMapper` as a base `Mapper` class.
|
11
|
+
- Add error stream for `RuntimeError` to `RecordImporter`.
|
12
|
+
- Add error stream handling for `Ldp::HTTPError` and `Faraday::Connection`
|
13
|
+
errors to `RecordImporter`.
|
14
|
+
- Add `info_stream`, `default_info_stream` (`#<<`) and notifications for
|
15
|
+
before/after record import.
|
16
|
+
- Improve validator documentation.
|
17
|
+
- Add `TitleValidator` to validate presence of titles in parsed
|
18
|
+
`InputRecord`s.
|
19
|
+
|
1
20
|
0.1.1 - Fri Jan 12, 2018
|
2
21
|
------------------------
|
3
22
|
|
data/README.md
CHANGED
@@ -3,6 +3,26 @@ Darlingtonia
|
|
3
3
|
|
4
4
|
Object import for Hyrax.
|
5
5
|
|
6
|
+
Usage
|
7
|
+
-----
|
8
|
+
|
9
|
+
In your project's `Gemfile`, add: `gem 'darlingtonia', '~> 0.1'`, then do `bundle install`.
|
10
|
+
|
11
|
+
|
12
|
+
This software is primarily intended for use in a [Hyrax](https://github.com/samvera/hyrax) project.
|
13
|
+
However, its dependency on `hyrax` is kept strictly optional so most of its code can be reused to
|
14
|
+
good effect elsewhere.
|
15
|
+
|
16
|
+
To do a basic Hyrax import, first ensure that a [work type is registered](http://www.rubydoc.info/github/samvera/hyrax/Hyrax/Configuration#register_curation_concern-instance_method)
|
17
|
+
with your `Hyrax` application. You need to provide a `Parser` (out of the box, we support simple CSV
|
18
|
+
import with `CsvParser`).
|
19
|
+
|
20
|
+
```ruby
|
21
|
+
parser = Darlingtonia::CsvParser.new(file: File.open('path/to/import.csv'))
|
22
|
+
|
23
|
+
Darlingtonia::Importer.new(parser: parser).import
|
24
|
+
```
|
25
|
+
|
6
26
|
Development
|
7
27
|
-----------
|
8
28
|
|
@@ -11,5 +31,5 @@ git clone https://github.com/curationexperts/darlingtonia
|
|
11
31
|
cd darlingtonia
|
12
32
|
|
13
33
|
bundle install
|
14
|
-
bundle exec
|
34
|
+
bundle exec rake ci
|
15
35
|
```
|
data/lib/darlingtonia.rb
CHANGED
@@ -3,9 +3,46 @@
|
|
3
3
|
require 'active_fedora'
|
4
4
|
|
5
5
|
##
|
6
|
-
# Bulk object import for Hyrax
|
6
|
+
# Bulk object import for Hyrax.
|
7
|
+
#
|
8
|
+
# @example A basic configuration
|
9
|
+
# Darlingtonia.config do |config|
|
10
|
+
# # error streams must respond to `#<<`
|
11
|
+
# config.default_error_stream = MyErrorStream.new
|
12
|
+
# end
|
13
|
+
#
|
7
14
|
module Darlingtonia
|
15
|
+
##
|
16
|
+
# @yield the current configuration
|
17
|
+
# @yieldparam config [Darlingtonia::Configuration]
|
18
|
+
#
|
19
|
+
# @return [Darlingtonia::Configuration] the current configuration
|
20
|
+
def config
|
21
|
+
yield @configuration if block_given?
|
22
|
+
@configuration
|
23
|
+
end
|
24
|
+
module_function :config
|
25
|
+
|
26
|
+
##
|
27
|
+
# Module-wide options for `Darlingtonia`.
|
28
|
+
class Configuration
|
29
|
+
##
|
30
|
+
# @!attribute [rw] default_error_stream
|
31
|
+
# @return [#<<]
|
32
|
+
# @!attribute [rw] default_info_stream
|
33
|
+
# @return [#<<]
|
34
|
+
attr_accessor :default_error_stream, :default_info_stream
|
35
|
+
|
36
|
+
def initialize
|
37
|
+
self.default_error_stream = STDOUT
|
38
|
+
self.default_info_stream = STDOUT
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
@configuration = Configuration.new
|
43
|
+
|
8
44
|
require 'darlingtonia/version'
|
45
|
+
require 'darlingtonia/metadata_mapper'
|
9
46
|
require 'darlingtonia/hash_mapper'
|
10
47
|
|
11
48
|
require 'darlingtonia/importer'
|
@@ -13,9 +50,10 @@ module Darlingtonia
|
|
13
50
|
|
14
51
|
require 'darlingtonia/input_record'
|
15
52
|
|
16
|
-
require 'darlingtonia/parser'
|
17
|
-
require 'darlingtonia/parsers/csv_parser'
|
18
|
-
|
19
53
|
require 'darlingtonia/validator'
|
20
54
|
require 'darlingtonia/validators/csv_format_validator'
|
55
|
+
require 'darlingtonia/validators/title_validator'
|
56
|
+
|
57
|
+
require 'darlingtonia/parser'
|
58
|
+
require 'darlingtonia/parsers/csv_parser'
|
21
59
|
end
|
@@ -5,12 +5,22 @@ module Darlingtonia
|
|
5
5
|
# A generic metadata mapper for input records
|
6
6
|
#
|
7
7
|
# Maps from hash accessor syntax (`['title']`) to method call dot syntax (`.title`)
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
8
|
+
#
|
9
|
+
# The fields provided by this mapper are dynamically determined by the fields
|
10
|
+
# available in the provided metadata hash.
|
11
|
+
#
|
12
|
+
# All field values are given as multi-valued arrays.
|
13
|
+
#
|
14
|
+
# @example
|
15
|
+
# mapper = HashMapper.new
|
16
|
+
# mapper.fields # => []
|
17
|
+
#
|
18
|
+
# mapper.metadata = { title: 'Comet in Moominland', author: 'Tove Jansson' }
|
19
|
+
# mapper.fields # => [:title, :author]
|
20
|
+
# mapper.title # => ['Comet in Moominland']
|
21
|
+
# mapper.author # => ['Tove Jansson']
|
22
|
+
#
|
23
|
+
class HashMapper < MetadataMapper
|
14
24
|
##
|
15
25
|
# @param meta [#to_h]
|
16
26
|
# @return [Hash<String, String>]
|
@@ -18,14 +28,6 @@ module Darlingtonia
|
|
18
28
|
@metadata = meta.to_h
|
19
29
|
end
|
20
30
|
|
21
|
-
##
|
22
|
-
# @param name [Symbol]
|
23
|
-
#
|
24
|
-
# @return [Boolean]
|
25
|
-
def field?(name)
|
26
|
-
fields.include?(name)
|
27
|
-
end
|
28
|
-
|
29
31
|
##
|
30
32
|
# @return [Enumerable<Symbol>] The fields the mapper can process
|
31
33
|
def fields
|
@@ -33,13 +35,10 @@ module Darlingtonia
|
|
33
35
|
metadata.keys.map(&:to_sym)
|
34
36
|
end
|
35
37
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
def respond_to_missing?(method_name, include_private = false)
|
42
|
-
field?(method_name) || super
|
38
|
+
##
|
39
|
+
# @see MetadataMapper#map_field
|
40
|
+
def map_field(name)
|
41
|
+
Array(metadata[name.to_s])
|
43
42
|
end
|
44
43
|
end
|
45
44
|
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Darlingtonia
|
4
|
+
##
|
5
|
+
# A null/base mapper that maps no fields.
|
6
|
+
#
|
7
|
+
# Real mapper implementations need to provide `#fields`, enumerating over
|
8
|
+
# `Symbols` that represent the fields on the target object (e.g. an
|
9
|
+
# `ActiveFedora::Base`/`Hyrax::WorkBehavior`) that the mapper can handle.
|
10
|
+
# For each field in `#fields`, the mapper must respond to a matching method
|
11
|
+
# (i.e. `:title` => `#title`), and return the value(s) that should be set to
|
12
|
+
# the target's attributes upon mapping.
|
13
|
+
#
|
14
|
+
# To ease the implementation of field methods, this base class provides
|
15
|
+
# a `#method_missing` that forwards missing method names to a `#map_field`
|
16
|
+
# method. `#map_field` can be implemented to provide a generalized field
|
17
|
+
# mapping when a common pattern will be used for many methods. Callers should
|
18
|
+
# avoid relying on this protected method directly, since mappers may implement
|
19
|
+
# individual field methods in any other way (e.g. `def title; end`) to route
|
20
|
+
# around `#map_field`. Implementations are also free to override
|
21
|
+
# `#method_missing` if desired.
|
22
|
+
#
|
23
|
+
# Mappers generally operate over some input `#metadata`. Example metadata
|
24
|
+
# types that mappers could be implemented over include `Hash`, `CSV`, `XML`,
|
25
|
+
# `RDF::Graph`, etc...; mappers are free to interpret or ignore the contents
|
26
|
+
# of their underlying metadata data structures at their leisure. Values for
|
27
|
+
# fields are /usually/ derived from the `#metadata`, but can also be generated
|
28
|
+
# from complex logic or even hard-coded.
|
29
|
+
#
|
30
|
+
# @example Using a MetadataMapper
|
31
|
+
# mapper = MyMapper.new
|
32
|
+
# mapper.metadata = some_metadata_object
|
33
|
+
# mapper.fields # => [:title, :author, :description]
|
34
|
+
#
|
35
|
+
# mapper.title # => 'Some Title'
|
36
|
+
#
|
37
|
+
# mapper.fields.map { |field| mapper.send(field) }
|
38
|
+
#
|
39
|
+
#
|
40
|
+
# @see ImportRecord#attributes for the canonical usage of a `MetadataMapper`.
|
41
|
+
# @see HashMapper for an example implementation with dynamically generated
|
42
|
+
# fields
|
43
|
+
class MetadataMapper
|
44
|
+
##
|
45
|
+
# @!attribute [rw] metadata
|
46
|
+
# @return [Object]
|
47
|
+
attr_accessor :metadata
|
48
|
+
|
49
|
+
##
|
50
|
+
# @param name [Symbol]
|
51
|
+
#
|
52
|
+
# @return [Boolean]
|
53
|
+
def field?(name)
|
54
|
+
fields.include?(name)
|
55
|
+
end
|
56
|
+
|
57
|
+
##
|
58
|
+
# @return [Enumerable<Symbol>] The fields the mapper can process
|
59
|
+
def fields
|
60
|
+
[]
|
61
|
+
end
|
62
|
+
|
63
|
+
def method_missing(method_name, *args, &block)
|
64
|
+
return map_field(method_name) if fields.include?(method_name)
|
65
|
+
super
|
66
|
+
end
|
67
|
+
|
68
|
+
def respond_to_missing?(method_name, include_private = false)
|
69
|
+
field?(method_name) || super
|
70
|
+
end
|
71
|
+
|
72
|
+
protected
|
73
|
+
|
74
|
+
##
|
75
|
+
# @private
|
76
|
+
#
|
77
|
+
# @param name [Symbol]
|
78
|
+
#
|
79
|
+
# @return [Object]
|
80
|
+
def map_field(_name)
|
81
|
+
raise NotImplementedError
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
data/lib/darlingtonia/parser.rb
CHANGED
@@ -2,13 +2,39 @@
|
|
2
2
|
|
3
3
|
module Darlingtonia
|
4
4
|
##
|
5
|
-
# A generic parser
|
5
|
+
# A generic parser.
|
6
6
|
#
|
7
|
-
#
|
7
|
+
# `Parser` implementations provide a stream of `InputRecord`s, derived from an
|
8
|
+
# input object (`file`), through `Parser#records`. This method should be
|
9
|
+
# implemented efficiently for repeated access, generating records lazily if
|
10
|
+
# possible, and caching if appropriate.
|
11
|
+
#
|
12
|
+
# Input validation is handled by an array of `#validators`, which are run in
|
13
|
+
# sequence when `#validate` (or `#validate!`) is called. Errors caught in
|
14
|
+
# validation are accessible via `#errors`, and inputs generating errors result
|
15
|
+
# in `#valid? # => false`.
|
16
|
+
#
|
17
|
+
# A factory method `.for` is provided, and each implementation should
|
18
|
+
# provides a `.match?(**)` which returns `true` if the options passed
|
19
|
+
# indicate the parser can handle the given input. Parsers are checked for
|
20
|
+
# `#match?` in the reverse of load order (i.e. the most recently loaded
|
21
|
+
# `Parser` classes are given precedence).
|
22
|
+
#
|
23
|
+
# @example Getting a parser for a file input
|
8
24
|
# file = File.open('path/to/import/manifest.csv')
|
9
25
|
#
|
10
26
|
# Parser.for(file: file).records
|
11
27
|
#
|
28
|
+
# @example Validating a parser
|
29
|
+
# parser = Parser.for(file: invalid_input)
|
30
|
+
#
|
31
|
+
# parser.valid? # => true (always true before validation)
|
32
|
+
# parser.validate # => false
|
33
|
+
# parser.valid? # => false
|
34
|
+
# parser.errors # => an array of Validation::Error-like structs
|
35
|
+
#
|
36
|
+
# parser.validate! # ValidationError
|
37
|
+
#
|
12
38
|
# rubocop:disable Style/ClassVars
|
13
39
|
class Parser
|
14
40
|
DEFAULT_VALIDATORS = [].freeze
|
@@ -29,7 +55,7 @@ module Darlingtonia
|
|
29
55
|
def initialize(file:, **_opts)
|
30
56
|
self.file = file
|
31
57
|
@errors = []
|
32
|
-
@validators ||=
|
58
|
+
@validators ||= self.class::DEFAULT_VALIDATORS
|
33
59
|
|
34
60
|
yield self if block_given?
|
35
61
|
end
|
@@ -77,13 +103,13 @@ module Darlingtonia
|
|
77
103
|
end
|
78
104
|
|
79
105
|
##
|
80
|
-
# @return [Boolean] true if the
|
106
|
+
# @return [Boolean] true if the file input is valid
|
81
107
|
def valid?
|
82
108
|
errors.empty?
|
83
109
|
end
|
84
110
|
|
85
111
|
##
|
86
|
-
# @return [Boolean]
|
112
|
+
# @return [Boolean] true if the file input is valid
|
87
113
|
def validate
|
88
114
|
validators.each_with_object(errors) do |validator, errs|
|
89
115
|
errs.concat(validator.validate(parser: self))
|
@@ -93,7 +119,8 @@ module Darlingtonia
|
|
93
119
|
end
|
94
120
|
|
95
121
|
##
|
96
|
-
# @return [true]
|
122
|
+
# @return [true] always true, unless an error is raised.
|
123
|
+
#
|
97
124
|
# @raise [ValidationError] if the file to parse is invalid
|
98
125
|
def validate!
|
99
126
|
validate || raise(ValidationError)
|
@@ -4,8 +4,16 @@ require 'csv'
|
|
4
4
|
|
5
5
|
module Darlingtonia
|
6
6
|
##
|
7
|
-
# A parser for CSV files
|
7
|
+
# A parser for CSV files. A single `InputRecord` is returned for each row
|
8
|
+
# parsed from the input.
|
9
|
+
#
|
10
|
+
# Validates the format of the CSV, generating a single error the file is
|
11
|
+
# malformed. This error gives the line number and a message for the first
|
12
|
+
# badly formatted row.
|
13
|
+
#
|
14
|
+
# @see CsvFormatValidator
|
8
15
|
class CsvParser < Parser
|
16
|
+
DEFAULT_VALIDATORS = [CsvFormatValidator.new].freeze
|
9
17
|
EXTENSION = '.csv'
|
10
18
|
|
11
19
|
class << self
|
@@ -2,12 +2,32 @@
|
|
2
2
|
|
3
3
|
module Darlingtonia
|
4
4
|
class RecordImporter
|
5
|
+
##
|
6
|
+
# @!attribute [rw] error_stream
|
7
|
+
# @return [#<<]
|
8
|
+
# @!attribute [rw] info_stream
|
9
|
+
# @return [#<<]
|
10
|
+
attr_accessor :error_stream, :info_stream
|
11
|
+
|
12
|
+
##
|
13
|
+
# @param error_stream [#<<]
|
14
|
+
def initialize(error_stream: Darlingtonia.config.default_error_stream,
|
15
|
+
info_stream: Darlingtonia.config.default_info_stream)
|
16
|
+
self.error_stream = error_stream
|
17
|
+
self.info_stream = info_stream
|
18
|
+
end
|
19
|
+
|
5
20
|
##
|
6
21
|
# @param record [ImportRecord]
|
7
22
|
#
|
8
23
|
# @return [void]
|
9
24
|
def import(record:)
|
10
|
-
|
25
|
+
create_for(record: record)
|
26
|
+
rescue Faraday::ConnectionFailed, Ldp::HttpError => e
|
27
|
+
error_stream << e
|
28
|
+
rescue RuntimeError => e
|
29
|
+
error_stream << e
|
30
|
+
raise e
|
11
31
|
end
|
12
32
|
|
13
33
|
def import_type
|
@@ -16,5 +36,16 @@ module Darlingtonia
|
|
16
36
|
|
17
37
|
Hyrax.config.curation_concerns.first
|
18
38
|
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def create_for(record:)
|
43
|
+
info_stream << 'Creating record: ' \
|
44
|
+
"#{record.respond_to?(:title) ? record.title : record}."
|
45
|
+
|
46
|
+
created = import_type.create(record.attributes)
|
47
|
+
|
48
|
+
info_stream << "Record created at: #{created.id}"
|
49
|
+
end
|
19
50
|
end
|
20
51
|
end
|
@@ -4,6 +4,19 @@ module Darlingtonia
|
|
4
4
|
##
|
5
5
|
# @abstract A null validator; always returns an empty error collection
|
6
6
|
#
|
7
|
+
# Validators are used to ensure the correctness of input to a parser. Each
|
8
|
+
# validator must respond to `#validate` and return a collection of errors
|
9
|
+
# found during validation. If the input is valid, this collection must be
|
10
|
+
# empty. Otherwise, it contains any number of `Validator::Error` structs
|
11
|
+
# which should be sent to the `#error_stream` by the validator.
|
12
|
+
#
|
13
|
+
# The validation process accepts an entire `Parser` and is free to inspect
|
14
|
+
# the input `#file` content, or view its individual `#records`.
|
15
|
+
#
|
16
|
+
# The base class provides infrastructure for the key behavior, relying on a
|
17
|
+
# private `#run_validation` method to provide the core behavior. In most cases
|
18
|
+
# implementers will want to simply override this method.
|
19
|
+
#
|
7
20
|
# @example validating a parser
|
8
21
|
# validator = MyValidator.new
|
9
22
|
# validator.validate(parser: myParser)
|
@@ -16,19 +29,66 @@ module Darlingtonia
|
|
16
29
|
# description: '...'
|
17
30
|
# lineno: 37>
|
18
31
|
#
|
32
|
+
# @example Implementing a custom Validator and using it in a Parser
|
33
|
+
# # Validator checks that the title, when downcased is equal to `moomin`
|
34
|
+
# class TitleIsMoominValidator
|
35
|
+
# def run_validation(parser:)
|
36
|
+
# parser.records.each_with_object([]) do |record, errors|
|
37
|
+
# errors << Error.new(self, :title_is_not_moomin) unless
|
38
|
+
# title_is_moomin?(record)
|
39
|
+
# end
|
40
|
+
# end
|
41
|
+
#
|
42
|
+
# def title_is_moomin?(record)
|
43
|
+
# return false unless record.respond_to?(:title)
|
44
|
+
# return true if record.title.downcase == 'moomin
|
45
|
+
# true
|
46
|
+
# end
|
47
|
+
# end
|
48
|
+
#
|
49
|
+
# parser = MyParser.new(some_content)
|
50
|
+
# parser.validations << TitleIsMoominvalidator.new
|
51
|
+
# parser.validate
|
52
|
+
# parser.valid? # => false (unless all the records match the title)
|
53
|
+
#
|
54
|
+
# @see Parser#validate
|
19
55
|
class Validator
|
20
|
-
Error = Struct.new(:validator, :name, :description, :lineno)
|
56
|
+
Error = Struct.new(:validator, :name, :description, :lineno) do
|
57
|
+
def to_s
|
58
|
+
"#{name}: #{description} (#{validator})"
|
59
|
+
end
|
60
|
+
end
|
21
61
|
|
22
62
|
##
|
23
|
-
#
|
24
|
-
#
|
63
|
+
# @!attribute [rw] error_stream
|
64
|
+
# @return [#<<]
|
65
|
+
attr_accessor :error_stream
|
66
|
+
|
67
|
+
##
|
68
|
+
# @param error_stream [#<<]
|
69
|
+
def initialize(error_stream: Darlingtonia.config.default_error_stream)
|
70
|
+
self.error_stream = error_stream
|
71
|
+
end
|
72
|
+
|
73
|
+
##
|
74
|
+
# @param parser [Parser]
|
25
75
|
#
|
26
76
|
# @return [Enumerator<Error>] a collection of errors found in validation
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
77
|
+
def validate(parser:)
|
78
|
+
run_validation(parser: parser).tap do |errors|
|
79
|
+
errors.map { |error| error_stream << error }
|
80
|
+
end
|
31
81
|
end
|
32
82
|
# rubocop:enable Lint/UnusedMethodArgument
|
83
|
+
|
84
|
+
private
|
85
|
+
|
86
|
+
##
|
87
|
+
# @return [Enumerator<Error>]
|
88
|
+
#
|
89
|
+
# rubocop:disable Lint/UnusedMethodArgument
|
90
|
+
def run_validation(parser:)
|
91
|
+
[].to_enum
|
92
|
+
end
|
33
93
|
end
|
34
94
|
end
|
@@ -12,8 +12,10 @@ module Darlingtonia
|
|
12
12
|
# @see http://ruby-doc.org/stdlib-2.0.0/libdoc/csv/rdoc/CSV/MalformedCSVError.html
|
13
13
|
class CsvFormatValidator < Validator
|
14
14
|
##
|
15
|
+
# @private
|
16
|
+
#
|
15
17
|
# @see Validator#validate
|
16
|
-
def
|
18
|
+
def run_validation(parser:, **)
|
17
19
|
return [] if CSV.parse(parser.file.read)
|
18
20
|
rescue CSV::MalformedCSVError => e
|
19
21
|
[Error.new(self.class, e.class, e.message)]
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Darlingtonia
|
4
|
+
class TitleValidator < Validator
|
5
|
+
##
|
6
|
+
# @private
|
7
|
+
#
|
8
|
+
# @see Validator#validate
|
9
|
+
def run_validation(parser:, **)
|
10
|
+
parser.records.each_with_object([]) do |record, errors|
|
11
|
+
titles = record.respond_to?(:title) ? record.title : []
|
12
|
+
|
13
|
+
errors << error_for(record: record) if Array(titles).empty?
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
protected
|
18
|
+
|
19
|
+
##
|
20
|
+
# @private
|
21
|
+
# @param record [InputRecord]
|
22
|
+
#
|
23
|
+
# @return [Error]
|
24
|
+
def error_for(record:)
|
25
|
+
Error.new(self,
|
26
|
+
:missing_title,
|
27
|
+
"Title is required; got #{record.mapper.metadata}")
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/lib/darlingtonia/version.rb
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
require 'spec_helper'
|
4
4
|
|
5
5
|
describe Darlingtonia::CsvFormatValidator do
|
6
|
-
subject(:validator) { described_class.new }
|
6
|
+
subject(:validator) { described_class.new(error_stream: []) }
|
7
7
|
let(:invalid_parser) { Darlingtonia::CsvParser.new(file: invalid_file) }
|
8
8
|
let(:invalid_file) { File.open('spec/fixtures/bad_example.csv') }
|
9
9
|
|
@@ -46,4 +46,18 @@ EOS
|
|
46
46
|
expect(parser.records.map(&:date)).to contain_exactly(['1945'], ['1946'])
|
47
47
|
end
|
48
48
|
end
|
49
|
+
|
50
|
+
describe '#validate' do
|
51
|
+
it 'is valid' do
|
52
|
+
expect(parser.validate).to be_truthy
|
53
|
+
end
|
54
|
+
|
55
|
+
context 'with invalid file' do
|
56
|
+
let(:file) { File.open('spec/fixtures/bad_example.csv') }
|
57
|
+
|
58
|
+
it 'is invalid' do
|
59
|
+
expect(parser.validate).to be_falsey
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
49
63
|
end
|
@@ -17,7 +17,7 @@ describe Darlingtonia::Parser do
|
|
17
17
|
before(:context) do
|
18
18
|
##
|
19
19
|
# An importer that matches all types
|
20
|
-
class
|
20
|
+
class MyFakeParser < described_class
|
21
21
|
class << self
|
22
22
|
def match?(**_opts)
|
23
23
|
true
|
@@ -25,11 +25,11 @@ describe Darlingtonia::Parser do
|
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
28
|
-
class NestedParser <
|
28
|
+
class NestedParser < MyFakeParser; end
|
29
29
|
end
|
30
30
|
|
31
31
|
after(:context) do
|
32
|
-
Object.send(:remove_const, :
|
32
|
+
Object.send(:remove_const, :MyFakeParser)
|
33
33
|
Object.send(:remove_const, :NestedParser)
|
34
34
|
end
|
35
35
|
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'spec_helper'
|
4
|
+
|
5
|
+
describe Darlingtonia::RecordImporter, :clean do
|
6
|
+
subject(:importer) do
|
7
|
+
described_class.new(error_stream: error_stream, info_stream: info_stream)
|
8
|
+
end
|
9
|
+
|
10
|
+
let(:error_stream) { [] }
|
11
|
+
let(:info_stream) { [] }
|
12
|
+
let(:record) { Darlingtonia::InputRecord.new }
|
13
|
+
|
14
|
+
it 'raises an error when no work type exists' do
|
15
|
+
expect { importer.import(record: record) }
|
16
|
+
.to raise_error 'No curation_concern found for import'
|
17
|
+
end
|
18
|
+
|
19
|
+
context 'with a registered work type' do
|
20
|
+
include_context 'with a work type'
|
21
|
+
|
22
|
+
it 'creates a work for record' do
|
23
|
+
expect { importer.import(record: record) }
|
24
|
+
.to change { Work.count }
|
25
|
+
.by 1
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'writes to the info stream before and after create' do
|
29
|
+
expect { importer.import(record: record) }
|
30
|
+
.to change { info_stream }
|
31
|
+
.to contain_exactly(/^Creating record/, /^Record created/)
|
32
|
+
end
|
33
|
+
|
34
|
+
context 'when input record errors with LDP errors' do
|
35
|
+
let(:ldp_error) { Ldp::PreconditionFailed }
|
36
|
+
|
37
|
+
before { allow(record).to receive(:attributes).and_raise(ldp_error) }
|
38
|
+
|
39
|
+
it 'writes errors to the error stream (no reraise!)' do
|
40
|
+
expect { importer.import(record: record) }
|
41
|
+
.to change { error_stream }
|
42
|
+
.to contain_exactly(an_instance_of(ldp_error))
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
context 'when input record errors unexpectedly' do
|
47
|
+
let(:custom_error) { Class.new(RuntimeError) }
|
48
|
+
|
49
|
+
before { allow(record).to receive(:attributes).and_raise(custom_error) }
|
50
|
+
|
51
|
+
it 'writes errors to the error stream' do
|
52
|
+
expect { begin; importer.import(record: record); rescue; end }
|
53
|
+
.to change { error_stream }
|
54
|
+
.to contain_exactly(an_instance_of(custom_error))
|
55
|
+
end
|
56
|
+
|
57
|
+
it 'reraises error' do
|
58
|
+
expect { importer.import(record: record) }.to raise_error(custom_error)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'spec_helper'
|
4
|
+
|
5
|
+
describe Darlingtonia::TitleValidator do
|
6
|
+
subject(:validator) { described_class.new(error_stream: []) }
|
7
|
+
|
8
|
+
let(:invalid_parser) do
|
9
|
+
FakeParser.new(file: [{ 'title' => 'moomin' }, {}, {}])
|
10
|
+
end
|
11
|
+
|
12
|
+
it_behaves_like 'a Darlingtonia::Validator' do
|
13
|
+
let(:valid_parser) { FakeParser.new(file: [{ 'title' => 'moomin' }]) }
|
14
|
+
end
|
15
|
+
|
16
|
+
describe '#validate' do
|
17
|
+
it 'populates errors for records with missing titles' do
|
18
|
+
expect(validator.validate(parser: invalid_parser))
|
19
|
+
.to contain_exactly(an_instance_of(described_class::Error),
|
20
|
+
an_instance_of(described_class::Error))
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'spec_helper'
|
4
|
+
|
5
|
+
describe Darlingtonia do
|
6
|
+
describe '#config' do
|
7
|
+
it 'can set a default error stream' do
|
8
|
+
stream = []
|
9
|
+
|
10
|
+
expect { described_class.config { |c| c.default_error_stream = stream } }
|
11
|
+
.to change { described_class.config.default_error_stream }
|
12
|
+
.from(STDOUT)
|
13
|
+
.to(stream)
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'can set a default info stream' do
|
17
|
+
stream = []
|
18
|
+
|
19
|
+
expect { described_class.config { |c| c.default_info_stream = stream } }
|
20
|
+
.to change { described_class.config.default_info_stream }
|
21
|
+
.from(STDOUT)
|
22
|
+
.to(stream)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -2,37 +2,26 @@
|
|
2
2
|
|
3
3
|
require 'spec_helper'
|
4
4
|
|
5
|
-
describe 'importing a csv batch' do
|
5
|
+
describe 'importing a csv batch', :clean do
|
6
6
|
subject(:importer) { Darlingtonia::Importer.new(parser: parser) }
|
7
7
|
let(:parser) { Darlingtonia::CsvParser.new(file: file) }
|
8
8
|
let(:file) { File.open('spec/fixtures/example.csv') }
|
9
9
|
|
10
|
-
|
11
|
-
before do
|
12
|
-
class Work < ActiveFedora::Base
|
13
|
-
property :title, predicate: ::RDF::URI('http://example.com/title')
|
14
|
-
property :description, predicate: ::RDF::URI('http://example.com/description')
|
15
|
-
end
|
16
|
-
|
17
|
-
module Hyrax
|
18
|
-
def self.config
|
19
|
-
Config.new
|
20
|
-
end
|
10
|
+
include_context 'with a work type'
|
21
11
|
|
22
|
-
|
23
|
-
|
24
|
-
[Work]
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
12
|
+
it 'creates a record for each CSV line' do
|
13
|
+
expect { importer.import }.to change { Work.count }.to 3
|
28
14
|
end
|
29
15
|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
end
|
16
|
+
describe 'validation' do
|
17
|
+
context 'with invalid CSV' do
|
18
|
+
let(:file) { File.open('spec/fixtures/bad_example.csv') }
|
34
19
|
|
35
|
-
|
36
|
-
|
20
|
+
it 'outputs invalid file notice to error stream' do
|
21
|
+
expect { parser.validate }
|
22
|
+
.to output(/^CSV::MalformedCSVError.*line 2/)
|
23
|
+
.to_stdout_from_any_process
|
24
|
+
end
|
25
|
+
end
|
37
26
|
end
|
38
27
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -4,6 +4,7 @@ require 'pry' unless ENV['CI']
|
|
4
4
|
ENV['environment'] ||= 'test'
|
5
5
|
|
6
6
|
require 'bundler/setup'
|
7
|
+
require 'active_fedora/cleaner'
|
7
8
|
require 'darlingtonia'
|
8
9
|
|
9
10
|
Dir['./spec/support/**/*.rb'].each { |f| require f }
|
@@ -11,4 +12,6 @@ Dir['./spec/support/**/*.rb'].each { |f| require f }
|
|
11
12
|
RSpec.configure do |config|
|
12
13
|
config.filter_run focus: true
|
13
14
|
config.run_all_when_everything_filtered = true
|
15
|
+
|
16
|
+
config.before(:each, clean: true) { ActiveFedora::Cleaner.clean! }
|
14
17
|
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
shared_context 'with a work type' do
|
4
|
+
# A work type must be defined for the default `RecordImporter` to save objects
|
5
|
+
before do
|
6
|
+
class Work < ActiveFedora::Base
|
7
|
+
property :title, predicate: ::RDF::URI('http://example.com/title')
|
8
|
+
property :description, predicate: ::RDF::URI('http://example.com/description')
|
9
|
+
end
|
10
|
+
|
11
|
+
module Hyrax
|
12
|
+
def self.config
|
13
|
+
Config.new
|
14
|
+
end
|
15
|
+
|
16
|
+
class Config
|
17
|
+
def curation_concerns
|
18
|
+
[Work]
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
after do
|
25
|
+
Object.send(:remove_const, :Hyrax) if defined?(Hyrax)
|
26
|
+
Object.send(:remove_const, :Work) if defined?(Work)
|
27
|
+
end
|
28
|
+
end
|
@@ -1,7 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
shared_examples 'a Darlingtonia::Validator' do
|
4
|
-
subject(:validator) { described_class.new }
|
4
|
+
subject(:validator) { described_class.new(error_stream: error_stream) }
|
5
|
+
let(:error_stream) { [] }
|
5
6
|
|
6
7
|
define :be_a_validator_error do # |expected|
|
7
8
|
match { false } # { |actual| some_condition }
|
@@ -15,13 +16,13 @@ shared_examples 'a Darlingtonia::Validator' do
|
|
15
16
|
end
|
16
17
|
|
17
18
|
it 'gives an empty error collection for a valid parser' do
|
18
|
-
expect(validator.validate(parser: valid_parser)).
|
19
|
+
expect(validator.validate(parser: valid_parser)).not_to be_any if
|
19
20
|
defined?(valid_parser)
|
20
21
|
end
|
21
22
|
|
22
23
|
context 'for an invalid parser' do
|
23
24
|
it 'gives an non-empty error collection' do
|
24
|
-
expect(validator.validate(parser: invalid_parser)).
|
25
|
+
expect(validator.validate(parser: invalid_parser)).to be_any if
|
25
26
|
defined?(invalid_parser)
|
26
27
|
end
|
27
28
|
|
@@ -32,6 +33,14 @@ shared_examples 'a Darlingtonia::Validator' do
|
|
32
33
|
expect(error).to be_a_validator_error
|
33
34
|
end
|
34
35
|
end
|
36
|
+
|
37
|
+
it 'writes errors to the error stream' do
|
38
|
+
if defined?(invalid_parser)
|
39
|
+
expect { validator.validate(parser: invalid_parser) }
|
40
|
+
.to change { error_stream }
|
41
|
+
.to include(an_instance_of(Darlingtonia::Validator::Error))
|
42
|
+
end
|
43
|
+
end
|
35
44
|
end
|
36
45
|
end
|
37
46
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: darlingtonia
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tom Johnson
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-01-
|
12
|
+
date: 2018-01-17 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: active-fedora
|
@@ -148,11 +148,13 @@ files:
|
|
148
148
|
- lib/darlingtonia/hash_mapper.rb
|
149
149
|
- lib/darlingtonia/importer.rb
|
150
150
|
- lib/darlingtonia/input_record.rb
|
151
|
+
- lib/darlingtonia/metadata_mapper.rb
|
151
152
|
- lib/darlingtonia/parser.rb
|
152
153
|
- lib/darlingtonia/parsers/csv_parser.rb
|
153
154
|
- lib/darlingtonia/record_importer.rb
|
154
155
|
- lib/darlingtonia/validator.rb
|
155
156
|
- lib/darlingtonia/validators/csv_format_validator.rb
|
157
|
+
- lib/darlingtonia/validators/title_validator.rb
|
156
158
|
- lib/darlingtonia/version.rb
|
157
159
|
- spec/darlingtonia/csv_format_validator_spec.rb
|
158
160
|
- spec/darlingtonia/csv_parser_spec.rb
|
@@ -160,13 +162,17 @@ files:
|
|
160
162
|
- spec/darlingtonia/importer_spec.rb
|
161
163
|
- spec/darlingtonia/input_record_spec.rb
|
162
164
|
- spec/darlingtonia/parser_spec.rb
|
165
|
+
- spec/darlingtonia/record_importer_spec.rb
|
166
|
+
- spec/darlingtonia/title_validator_spec.rb
|
163
167
|
- spec/darlingtonia/validator_spec.rb
|
164
168
|
- spec/darlingtonia/version_spec.rb
|
169
|
+
- spec/darlingtonia_spec.rb
|
165
170
|
- spec/fixtures/bad_example.csv
|
166
171
|
- spec/fixtures/example.csv
|
167
172
|
- spec/integration/import_csv_spec.rb
|
168
173
|
- spec/spec_helper.rb
|
169
174
|
- spec/support/fakes/fake_parser.rb
|
175
|
+
- spec/support/shared_contexts/with_work_type.rb
|
170
176
|
- spec/support/shared_examples/a_mapper.rb
|
171
177
|
- spec/support/shared_examples/a_parser.rb
|
172
178
|
- spec/support/shared_examples/a_validator.rb
|