darlingtonia 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +2 -0
- data/.rubocop.yml +13 -0
- data/.travis.yml +14 -0
- data/Gemfile +12 -0
- data/README.md +15 -0
- data/Rakefile +34 -0
- data/darlingtonia.gemspec +33 -0
- data/lib/darlingtonia.rb +21 -0
- data/lib/darlingtonia/always_invalid_validator.rb +17 -0
- data/lib/darlingtonia/hash_mapper.rb +45 -0
- data/lib/darlingtonia/importer.rb +32 -0
- data/lib/darlingtonia/input_record.rb +45 -0
- data/lib/darlingtonia/parser.rb +103 -0
- data/lib/darlingtonia/parsers/csv_parser.rb +35 -0
- data/lib/darlingtonia/record_importer.rb +20 -0
- data/lib/darlingtonia/validator.rb +21 -0
- data/lib/darlingtonia/validators/csv_format_validator.rb +8 -0
- data/lib/darlingtonia/version.rb +5 -0
- data/spec/darlingtonia/csv_format_validator_spec.rb +7 -0
- data/spec/darlingtonia/csv_parser_spec.rb +48 -0
- data/spec/darlingtonia/hash_mapper_spec.rb +8 -0
- data/spec/darlingtonia/importer_spec.rb +41 -0
- data/spec/darlingtonia/input_record_spec.rb +39 -0
- data/spec/darlingtonia/parser_spec.rb +42 -0
- data/spec/darlingtonia/validator_spec.rb +7 -0
- data/spec/darlingtonia/version_spec.rb +7 -0
- data/spec/fixtures/example.csv +4 -0
- data/spec/integration/import_csv_spec.rb +38 -0
- data/spec/spec_helper.rb +14 -0
- data/spec/support/fakes/fake_parser.rb +26 -0
- data/spec/support/shared_examples/a_mapper.rb +32 -0
- data/spec/support/shared_examples/a_parser.rb +73 -0
- data/spec/support/shared_examples/a_validator.rb +37 -0
- metadata +195 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: a2fc4e8abdff2180e2ef7ff1303661e806c4c304
|
4
|
+
data.tar.gz: 7d0f74053c870263e63ff9b84dc72f77c3f923eb
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 752c341d5200f4bdb1c9a59a3f0188a31522558d20ea6e0d2c7f01eec36e4581aea4e6b6ffbc73c6fbc3dbce81db17e0af80a1c29ddf54bc7b0d453567f36351
|
7
|
+
data.tar.gz: 2ebc1641f150f1e34316a4a60f6fbb16d8fd6cf13a3a5a624f9b0d92707817662df32cdf8ff20512c4efb4ce9924f6c16fde9cef9f674174c6ab3f6483109771
|
data/.gitignore
ADDED
data/.rubocop.yml
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
data/Rakefile
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
begin
|
4
|
+
require 'bundler/setup'
|
5
|
+
rescue LoadError
|
6
|
+
puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
|
7
|
+
end
|
8
|
+
|
9
|
+
require 'fcrepo_wrapper'
|
10
|
+
require 'solr_wrapper'
|
11
|
+
require 'active_fedora/rake_support'
|
12
|
+
|
13
|
+
require 'rspec/core/rake_task'
|
14
|
+
require 'rubocop/rake_task'
|
15
|
+
|
16
|
+
Bundler::GemHelper.install_tasks
|
17
|
+
|
18
|
+
desc 'Run style checker'
|
19
|
+
RuboCop::RakeTask.new(:rubocop) do |task|
|
20
|
+
task.fail_on_error = true
|
21
|
+
end
|
22
|
+
|
23
|
+
desc 'Run specs'
|
24
|
+
RSpec::Core::RakeTask.new(:spec)
|
25
|
+
|
26
|
+
desc 'Run specs with Fedora & Solr servers'
|
27
|
+
task :spec_with_server do
|
28
|
+
with_test_server { Rake::Task['spec'].invoke }
|
29
|
+
end
|
30
|
+
|
31
|
+
desc 'Check style and run specs'
|
32
|
+
task ci: %w[rubocop spec_with_server]
|
33
|
+
|
34
|
+
task default: :ci
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
$LOAD_PATH.push File.expand_path('../lib', __FILE__)
|
4
|
+
require 'darlingtonia/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |gem|
|
7
|
+
gem.name = 'darlingtonia'
|
8
|
+
gem.version = Darlingtonia::VERSION
|
9
|
+
gem.platform = Gem::Platform::RUBY
|
10
|
+
gem.authors = ['Tom Johnson', 'Data Curation Experts']
|
11
|
+
gem.email = 'tom@curationexperts.com'
|
12
|
+
gem.summary = 'Hyrax importers.'
|
13
|
+
gem.license = 'Apache-2.0'
|
14
|
+
gem.files = %w[AUTHORS CHANGELOG.md README.md LICENSE] +
|
15
|
+
Dir.glob('lib/**/*.rb')
|
16
|
+
gem.require_paths = %w[lib]
|
17
|
+
gem.has_rdoc = false
|
18
|
+
|
19
|
+
gem.required_ruby_version = '>= 2.3.4'
|
20
|
+
|
21
|
+
gem.add_dependency 'active-fedora', '>= 11.0', '<= 12.99'
|
22
|
+
|
23
|
+
gem.add_development_dependency 'yard', '~> 0.9'
|
24
|
+
gem.add_development_dependency 'bixby', '~> 0.3'
|
25
|
+
gem.add_development_dependency 'hyrax-spec', '~> 0.2'
|
26
|
+
gem.add_development_dependency 'rspec', '~> 3.6'
|
27
|
+
gem.add_development_dependency 'coveralls', '~> 0.8'
|
28
|
+
gem.add_development_dependency 'solr_wrapper', '~> 0.3'
|
29
|
+
gem.add_development_dependency 'fcrepo_wrapper', '~> 0.9'
|
30
|
+
|
31
|
+
gem.files = `git ls-files`.split("\n")
|
32
|
+
gem.test_files = `git ls-files -- {spec}/*`.split("\n")
|
33
|
+
end
|
data/lib/darlingtonia.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_fedora'
|
4
|
+
|
5
|
+
##
|
6
|
+
# Bulk object import for Hyrax
|
7
|
+
module Darlingtonia
|
8
|
+
require 'darlingtonia/version'
|
9
|
+
require 'darlingtonia/hash_mapper'
|
10
|
+
|
11
|
+
require 'darlingtonia/importer'
|
12
|
+
require 'darlingtonia/record_importer'
|
13
|
+
|
14
|
+
require 'darlingtonia/input_record'
|
15
|
+
|
16
|
+
require 'darlingtonia/parser'
|
17
|
+
require 'darlingtonia/parsers/csv_parser'
|
18
|
+
|
19
|
+
require 'darlingtonia/validator'
|
20
|
+
require 'darlingtonia/validators/csv_format_validator'
|
21
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Darlingtonia
|
4
|
+
##
|
5
|
+
# A Validator that always gives an error named `:everytime`.
|
6
|
+
#
|
7
|
+
# @example
|
8
|
+
# validator = AlwaysInvalidValidator.new
|
9
|
+
# validator.validate(:anything, :at, :all) # => [Error<#...>]
|
10
|
+
class AlwaysInvalidValidator < Validator
|
11
|
+
##
|
12
|
+
# @return [Array<Validator::Error>]
|
13
|
+
def validate(*)
|
14
|
+
[Error.new(self, :everytime)]
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Darlingtonia
|
4
|
+
##
|
5
|
+
# A generic metadata mapper for input records
|
6
|
+
#
|
7
|
+
# Maps from hash accessor syntax (`['title']`) to method call dot syntax (`.title`)
|
8
|
+
class HashMapper
|
9
|
+
##
|
10
|
+
# @!attribute [r] meadata
|
11
|
+
# @return [Hash<String, String>]
|
12
|
+
attr_reader :metadata
|
13
|
+
|
14
|
+
##
|
15
|
+
# @param meta [#to_h]
|
16
|
+
# @return [Hash<String, String>]
|
17
|
+
def metadata=(meta)
|
18
|
+
@metadata = meta.to_h
|
19
|
+
end
|
20
|
+
|
21
|
+
##
|
22
|
+
# @param name [Symbol]
|
23
|
+
#
|
24
|
+
# @return [Boolean]
|
25
|
+
def field?(name)
|
26
|
+
fields.include?(name)
|
27
|
+
end
|
28
|
+
|
29
|
+
##
|
30
|
+
# @return [Enumerable<Symbol>] The fields the mapper can process
|
31
|
+
def fields
|
32
|
+
return [] if metadata.nil?
|
33
|
+
metadata.keys.map(&:to_sym)
|
34
|
+
end
|
35
|
+
|
36
|
+
def method_missing(method_name, *args, &block)
|
37
|
+
return Array(metadata[method_name.to_s]) if fields.include?(method_name)
|
38
|
+
super
|
39
|
+
end
|
40
|
+
|
41
|
+
def respond_to_missing?(method_name, include_private = false)
|
42
|
+
field?(method_name) || super
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Darlingtonia
|
4
|
+
class Importer
|
5
|
+
extend Forwardable
|
6
|
+
|
7
|
+
##
|
8
|
+
# @!attribute [rw] parser
|
9
|
+
# @return [Parser]
|
10
|
+
# @!attribute [rw] record_importer
|
11
|
+
# @return [RecordImporter]
|
12
|
+
attr_accessor :parser, :record_importer
|
13
|
+
|
14
|
+
##
|
15
|
+
# @!method records()
|
16
|
+
# @see Parser#records
|
17
|
+
def_delegator :parser, :records, :records
|
18
|
+
|
19
|
+
##
|
20
|
+
# @param parser [Parser]
|
21
|
+
def initialize(parser:, record_importer: RecordImporter.new)
|
22
|
+
self.parser = parser
|
23
|
+
self.record_importer = record_importer
|
24
|
+
end
|
25
|
+
|
26
|
+
##
|
27
|
+
# @return [void]
|
28
|
+
def import
|
29
|
+
records.each { |record| record_importer.import(record: record) }
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Darlingtonia
|
4
|
+
class InputRecord
|
5
|
+
##
|
6
|
+
# @!attribute [rw] mapper
|
7
|
+
# @return [#map_fields]
|
8
|
+
attr_accessor :mapper
|
9
|
+
|
10
|
+
##
|
11
|
+
# @param metadata [Object]
|
12
|
+
# @param mapper [#map_fields]
|
13
|
+
def initialize(mapper: HashMapper.new)
|
14
|
+
self.mapper = mapper
|
15
|
+
end
|
16
|
+
|
17
|
+
class << self
|
18
|
+
def from(metadata:, mapper: HashMapper.new)
|
19
|
+
mapper.metadata = metadata
|
20
|
+
new(mapper: mapper)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
##
|
25
|
+
# @return [Hash<Symbol, Object>]
|
26
|
+
def attributes
|
27
|
+
mapper.fields.each_with_object({}) do |field, attrs|
|
28
|
+
attrs[field] = public_send(field)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
##
|
33
|
+
# Respond to methods matching mapper fields
|
34
|
+
def method_missing(method_name, *args, &block)
|
35
|
+
return super unless mapper.field?(method_name)
|
36
|
+
mapper.public_send(method_name)
|
37
|
+
end
|
38
|
+
|
39
|
+
##
|
40
|
+
# @see #method_missing
|
41
|
+
def respond_to_missing?(method_name, include_private = false)
|
42
|
+
mapper.field?(method_name) || super
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Darlingtonia
|
4
|
+
##
|
5
|
+
# A generic parser
|
6
|
+
#
|
7
|
+
# @example
|
8
|
+
# file = File.open('path/to/import/manifest.csv')
|
9
|
+
#
|
10
|
+
# Parser.for(file: file).records
|
11
|
+
#
|
12
|
+
class Parser
|
13
|
+
@subclasses = [] # @private
|
14
|
+
|
15
|
+
##
|
16
|
+
# @!attribute [rw] file
|
17
|
+
# @return [File]
|
18
|
+
# @!attribute [rw] validators
|
19
|
+
# @return [Array<Validator>]
|
20
|
+
# @!attribute [r] errors
|
21
|
+
# @return [Array]
|
22
|
+
attr_accessor :file, :validators
|
23
|
+
attr_reader :errors
|
24
|
+
|
25
|
+
##
|
26
|
+
# @param file [File]
|
27
|
+
def initialize(file:, **_opts)
|
28
|
+
self.file = file
|
29
|
+
@errors = []
|
30
|
+
@validators ||= []
|
31
|
+
|
32
|
+
yield self if block_given?
|
33
|
+
end
|
34
|
+
|
35
|
+
class << self
|
36
|
+
##
|
37
|
+
# @param file [Object]
|
38
|
+
#
|
39
|
+
# @return [Darlingtonia::Parser] a parser instance appropriate for
|
40
|
+
# the arguments
|
41
|
+
#
|
42
|
+
# @raise [NoParserError]
|
43
|
+
def for(file:)
|
44
|
+
klass =
|
45
|
+
@subclasses.find { |k| k.match?(file: file) } ||
|
46
|
+
raise(NoParserError)
|
47
|
+
|
48
|
+
klass.new(file: file)
|
49
|
+
end
|
50
|
+
|
51
|
+
##
|
52
|
+
# @abstract
|
53
|
+
# @return [Boolean]
|
54
|
+
def match?(**_opts); end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
##
|
59
|
+
# @private Register a new class when inherited
|
60
|
+
def inherited(subclass)
|
61
|
+
@subclasses << subclass
|
62
|
+
super
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
##
|
67
|
+
# @abstract
|
68
|
+
#
|
69
|
+
# @yield [record] gives each record in the file to the block
|
70
|
+
# @yieldparam record [ImportRecord]
|
71
|
+
#
|
72
|
+
# @return [Enumerable<ImportRecord>]
|
73
|
+
def records
|
74
|
+
raise NotImplementedError
|
75
|
+
end
|
76
|
+
|
77
|
+
##
|
78
|
+
# @return [Boolean] true if the
|
79
|
+
def valid?
|
80
|
+
errors.empty?
|
81
|
+
end
|
82
|
+
|
83
|
+
##
|
84
|
+
# @return [Boolean]
|
85
|
+
def validate
|
86
|
+
validators.each_with_object(errors) do |validator, errs|
|
87
|
+
errs.concat(validator.validate(parser: self))
|
88
|
+
end
|
89
|
+
|
90
|
+
valid?
|
91
|
+
end
|
92
|
+
|
93
|
+
##
|
94
|
+
# @return [true]
|
95
|
+
# @raise [ValidationError] if the file to parse is invalid
|
96
|
+
def validate!
|
97
|
+
validate || raise(ValidationError)
|
98
|
+
end
|
99
|
+
|
100
|
+
class NoParserError < TypeError; end
|
101
|
+
class ValidationError < RuntimeError; end
|
102
|
+
end
|
103
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'csv'
|
4
|
+
|
5
|
+
module Darlingtonia
|
6
|
+
##
|
7
|
+
# A parser for CSV files
|
8
|
+
class CsvParser < Parser
|
9
|
+
EXTENSION = '.csv'
|
10
|
+
|
11
|
+
class << self
|
12
|
+
##
|
13
|
+
# Matches all '.csv' filenames.
|
14
|
+
def match?(file:, **_opts)
|
15
|
+
File.extname(file) == EXTENSION
|
16
|
+
rescue TypeError
|
17
|
+
false
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
##
|
22
|
+
# Gives a record for each line in the .csv
|
23
|
+
#
|
24
|
+
# @see Parser#records
|
25
|
+
def records
|
26
|
+
return enum_for(:records) unless block_given?
|
27
|
+
|
28
|
+
file.rewind
|
29
|
+
|
30
|
+
CSV.parse(file.read, headers: true).each do |row|
|
31
|
+
yield InputRecord.from(metadata: row)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Darlingtonia
|
4
|
+
class RecordImporter
|
5
|
+
##
|
6
|
+
# @param record [ImportRecord]
|
7
|
+
#
|
8
|
+
# @return [void]
|
9
|
+
def import(record:)
|
10
|
+
import_type.create(record.attributes)
|
11
|
+
end
|
12
|
+
|
13
|
+
def import_type
|
14
|
+
raise 'No curation_concern found for import' unless
|
15
|
+
defined?(Hyrax) && Hyrax&.config&.curation_concerns&.any?
|
16
|
+
|
17
|
+
Hyrax.config.curation_concerns.first
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Darlingtonia
|
4
|
+
##
|
5
|
+
# @abstract A null validator; always returns an empty error collection
|
6
|
+
class Validator
|
7
|
+
Error = Struct.new(:validator, :name, :description, :lineno)
|
8
|
+
|
9
|
+
##
|
10
|
+
# @param parser [Parser]
|
11
|
+
# @param error_stream [#add]
|
12
|
+
#
|
13
|
+
# @return [Enumerator<Error>] a collection of errors found in validation
|
14
|
+
#
|
15
|
+
# rubocop:disable Lint/UnusedMethodArgument
|
16
|
+
def validate(parser:, **)
|
17
|
+
[]
|
18
|
+
end
|
19
|
+
# rubocop:enable Lint/UnusedMethodArgument
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require 'spec_helper'
|
5
|
+
|
6
|
+
describe Darlingtonia::CsvParser do
|
7
|
+
subject(:parser) { described_class.new(file: file) }
|
8
|
+
let(:file) { Tempfile.new(['fake', '.csv']) }
|
9
|
+
|
10
|
+
shared_context 'with content' do
|
11
|
+
let(:csv_content) do
|
12
|
+
<<-EOS
|
13
|
+
title,description,date
|
14
|
+
The Moomins and the Great Flood,"The Moomins and the Great Flood (Swedish: Småtrollen och den stora översvämningen, literally The Little Trolls and the Great Flood) is a book written by Finnish author Tove Jansson in 1945, during the end of World War II. It was the first book to star the Moomins, but is often seen as a prelude to the main Moomin books, as most of the main characters are introduced in the next book.",1945
|
15
|
+
Comet in Moominland,"Comet in Moominland is the second in Tove Jansson's series of Moomin books. Published in 1946, it marks the first appearance of several main characters, like Snufkin and the Snork Maiden.",1946
|
16
|
+
EOS
|
17
|
+
end
|
18
|
+
|
19
|
+
let(:record_count) { 2 }
|
20
|
+
|
21
|
+
before do
|
22
|
+
file.write(csv_content)
|
23
|
+
file.rewind
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
it_behaves_like 'a Darlingtonia::Parser' do
|
28
|
+
include_context 'with content'
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'matches .csv files' do
|
32
|
+
expect(Darlingtonia::Parser.for(file: file)).to be_a described_class
|
33
|
+
end
|
34
|
+
|
35
|
+
describe '#records' do
|
36
|
+
include_context 'with content'
|
37
|
+
|
38
|
+
it 'has the correct titles' do
|
39
|
+
expect(parser.records.map(&:title))
|
40
|
+
.to contain_exactly(['The Moomins and the Great Flood'],
|
41
|
+
['Comet in Moominland'])
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'has correct other fields' do
|
45
|
+
expect(parser.records.map(&:date)).to contain_exactly(['1945'], ['1946'])
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'spec_helper'
|
4
|
+
|
5
|
+
describe Darlingtonia::Importer do
|
6
|
+
subject(:importer) { described_class.new(parser: parser) }
|
7
|
+
let(:parser) { FakeParser.new(file: input) }
|
8
|
+
let(:input) { [{ 'title' => '1' }, { 'title' => '2' }, { 'title' => '3' }] }
|
9
|
+
|
10
|
+
let(:fake_record_importer) do
|
11
|
+
Class.new do
|
12
|
+
def import(record:)
|
13
|
+
records << record.attributes
|
14
|
+
end
|
15
|
+
|
16
|
+
def records
|
17
|
+
@records ||= []
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe '#records' do
|
23
|
+
it 'reflects the parsed records' do
|
24
|
+
expect(importer.records.map(&:attributes))
|
25
|
+
.to contain_exactly(*parser.records.map(&:attributes))
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
describe '#import' do
|
30
|
+
let(:record_importer) { fake_record_importer.new }
|
31
|
+
|
32
|
+
before { importer.record_importer = record_importer }
|
33
|
+
|
34
|
+
it 'sends records to the record importer' do
|
35
|
+
expect { importer.import }
|
36
|
+
.to change { record_importer.records }
|
37
|
+
.from(be_empty)
|
38
|
+
.to a_collection_containing_exactly(*importer.records.map(&:attributes))
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
describe Darlingtonia::InputRecord do
|
4
|
+
subject(:record) { described_class.from(metadata: metadata) }
|
5
|
+
|
6
|
+
let(:metadata) do
|
7
|
+
{ 'title' => 'Comet in Moominland',
|
8
|
+
'description' => 'A book about moomins.' }
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'has metadata and a mapper' do
|
12
|
+
is_expected
|
13
|
+
.to have_attributes(mapper: an_instance_of(Darlingtonia::HashMapper))
|
14
|
+
end
|
15
|
+
|
16
|
+
describe '#attributes' do
|
17
|
+
it 'handles basic text fields' do
|
18
|
+
expect(record.attributes).to include(:title, :description)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe 'mapped fields' do
|
23
|
+
it 'has methods for metadata fields' do
|
24
|
+
expect(record.title).to contain_exactly metadata['title']
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'has methods for additional mapped metadata fields' do
|
28
|
+
expect(record.description).to contain_exactly metadata['description']
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'knows it responds to methods for metadata fields' do
|
32
|
+
expect(record).to respond_to :title
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'knows it responds to methods for additional metadata fields' do
|
36
|
+
expect(record).to respond_to :description
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'spec_helper'
|
4
|
+
|
5
|
+
describe Darlingtonia::Parser do
|
6
|
+
subject(:parser) { described_class.new(file: file) }
|
7
|
+
let(:file) { :fake_file }
|
8
|
+
|
9
|
+
it_behaves_like 'a Darlingtonia::Parser'
|
10
|
+
|
11
|
+
describe '.for' do
|
12
|
+
it 'raises an error' do
|
13
|
+
expect { described_class.for(file: file) }.to raise_error TypeError
|
14
|
+
end
|
15
|
+
|
16
|
+
context 'with a matching parser subclass' do
|
17
|
+
before(:context) do
|
18
|
+
##
|
19
|
+
# An importer that matches all types
|
20
|
+
class FakeParser < described_class
|
21
|
+
class << self
|
22
|
+
def match?(**_opts)
|
23
|
+
true
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
after(:context) { Object.send(:remove_const, :FakeParser) }
|
30
|
+
|
31
|
+
it 'returns an importer instance' do
|
32
|
+
expect(described_class.for(file: file)).to be_a FakeParser
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
describe '#records' do
|
38
|
+
it 'raises NotImplementedError' do
|
39
|
+
expect { parser.records }.to raise_error NotImplementedError
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,4 @@
|
|
1
|
+
title,description
|
2
|
+
Fake Item,A description of a fake item.
|
3
|
+
Fake Item with Quoted Description,"Lorem ipsum dolor sit amet, cu mel habeo antiopam, id pri mucius oporteat. No graeco accumsan deterruisset est. Vix te sonet doctus perpetua, mei at odio eius nostrum. Ex postea quidam menandri duo. Rebum ullum cu mel.\nAperiam malorum indoctum ad nec.\nIn duo nonumes accusata, detraxit adipisci philosophia quo id. Ei usu volutpat vituperatoribus. Ut veniam dolorem qui. Ei legendos erroribus usu. Mentitum moderatius ei est, mei eius magna alterum no. Legere vivendum per ad, vim ei putent facilis."
|
4
|
+
Fake Item with Unicode Description,"Лорем ипсум долор сит амет, ех мовет толлит модератиус ест. Еу яуидам сенсерит цонсецтетуер про, при иисяуе ерудити цоррумпит ат. Ех усу нусяуам пхаедрум темпорибус, ест ат омнесяуе инструцтиор.\nЯуо ех мелиоре инсоленс праесент, иудицо тантас еурипидис хис ут. Аццусам урбанитас инструцтиор ан еам. Но хас вениам дицунт дебитис, нец ут суас аццусам перицула, нец риденс аетерно виртуте не.\nАт про еним вереар, ут солум юсто меи."
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'spec_helper'
|
4
|
+
|
5
|
+
describe 'importing a csv batch' do
|
6
|
+
subject(:importer) { Darlingtonia::Importer.new(parser: parser) }
|
7
|
+
let(:parser) { Darlingtonia::CsvParser.new(file: file) }
|
8
|
+
let(:file) { File.open('spec/fixtures/example.csv') }
|
9
|
+
|
10
|
+
# A work type must be defined for the default `RecordImporter` to save objects
|
11
|
+
before do
|
12
|
+
class Work < ActiveFedora::Base
|
13
|
+
property :title, predicate: ::RDF::URI('http://example.com/title')
|
14
|
+
property :description, predicate: ::RDF::URI('http://example.com/description')
|
15
|
+
end
|
16
|
+
|
17
|
+
module Hyrax
|
18
|
+
def self.config
|
19
|
+
Config.new
|
20
|
+
end
|
21
|
+
|
22
|
+
class Config
|
23
|
+
def curation_concerns
|
24
|
+
[Work]
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
after do
|
31
|
+
Object.send(:remove_const, :Hyrax)
|
32
|
+
Object.send(:remove_const, :Work)
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'creates a record for each CSV line' do
|
36
|
+
expect { importer.import }.to change { Work.count }.to 3
|
37
|
+
end
|
38
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'pry' unless ENV['CI']
|
4
|
+
ENV['environment'] ||= 'test'
|
5
|
+
|
6
|
+
require 'bundler/setup'
|
7
|
+
require 'darlingtonia'
|
8
|
+
|
9
|
+
Dir['./spec/support/**/*.rb'].each { |f| require f }
|
10
|
+
|
11
|
+
RSpec.configure do |config|
|
12
|
+
config.filter_run focus: true
|
13
|
+
config.run_all_when_everything_filtered = true
|
14
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class FakeParser < Darlingtonia::Parser
|
4
|
+
METADATA = [{ 'title' => '1' }, { 'title' => '2' }, { 'title' => '3' }].freeze
|
5
|
+
|
6
|
+
def initialize(file: METADATA)
|
7
|
+
super
|
8
|
+
end
|
9
|
+
|
10
|
+
def records
|
11
|
+
return enum_for(:records) unless block_given?
|
12
|
+
|
13
|
+
file.each { |hsh| yield Darlingtonia::InputRecord.from(metadata: hsh) }
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
require 'support/shared_examples/a_parser'
|
18
|
+
|
19
|
+
# rubocop:disable RSpec/FilePath
|
20
|
+
describe FakeParser do
|
21
|
+
it_behaves_like 'a Darlingtonia::Parser' do
|
22
|
+
subject(:parser) { described_class.new }
|
23
|
+
let(:record_count) { 3 }
|
24
|
+
end
|
25
|
+
end
|
26
|
+
# rubocop:enable RSpec/FilePath
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
shared_examples 'a Darlingtonia::Mapper' do
|
4
|
+
subject(:mapper) { described_class.new }
|
5
|
+
|
6
|
+
before { mapper.metadata = metadata }
|
7
|
+
|
8
|
+
describe '#metadata' do
|
9
|
+
it 'can be set' do
|
10
|
+
expect { mapper.metadata = nil }
|
11
|
+
.to change { mapper.metadata }
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
describe '#field?' do
|
16
|
+
it 'does not have bogus fields' do
|
17
|
+
expect(mapper.field?(:NOT_A_REAL_FIELD)).to be_falsey
|
18
|
+
end
|
19
|
+
|
20
|
+
it 'has fields that are expected' do
|
21
|
+
if defined?(expected_fields)
|
22
|
+
expected_fields.each do |field|
|
23
|
+
expect(mapper.field?(field)).to be_truthy
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
describe '#fields' do
|
30
|
+
it { expect(mapper.fields).to contain_exactly(*expected_fields) }
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'darlingtonia/always_invalid_validator'
|
4
|
+
|
5
|
+
shared_examples 'a Darlingtonia::Parser' do
|
6
|
+
describe '#file' do
|
7
|
+
it 'is an accessor' do
|
8
|
+
expect { parser.file = :a_new_file }
|
9
|
+
.to change { parser.file }
|
10
|
+
.to(:a_new_file)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
describe '#records' do
|
15
|
+
it 'yields records' do
|
16
|
+
unless described_class == Darlingtonia::Parser
|
17
|
+
expect { |b| parser.records(&b) }
|
18
|
+
.to yield_control.exactly(record_count).times
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
describe '#valid?' do
|
24
|
+
it 'is valid' do
|
25
|
+
expect(parser).to be_valid
|
26
|
+
end
|
27
|
+
|
28
|
+
context 'when not valid' do
|
29
|
+
before do
|
30
|
+
parser.validators = [Darlingtonia::AlwaysInvalidValidator.new]
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'is invalid' do
|
34
|
+
expect { parser.validate }
|
35
|
+
.to change { parser.valid? }
|
36
|
+
.to be_falsey
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
describe '#validate' do
|
42
|
+
it 'is true when valid' do
|
43
|
+
expect(parser.validate).to be_truthy
|
44
|
+
end
|
45
|
+
|
46
|
+
context 'when not valid' do
|
47
|
+
before do
|
48
|
+
parser.validators = [Darlingtonia::AlwaysInvalidValidator.new]
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'is invalid' do
|
52
|
+
expect(parser.validate).to be_falsey
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
describe '#validate!' do
|
58
|
+
it 'is true when valid' do
|
59
|
+
expect(parser.validate).to be_truthy
|
60
|
+
end
|
61
|
+
|
62
|
+
context 'when not valid' do
|
63
|
+
before do
|
64
|
+
parser.validators = [Darlingtonia::AlwaysInvalidValidator.new]
|
65
|
+
end
|
66
|
+
|
67
|
+
it 'raises a ValidationError' do
|
68
|
+
expect { parser.validate! }
|
69
|
+
.to raise_error Darlingtonia::Parser::ValidationError
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
shared_examples 'a Darlingtonia::Validator' do
|
4
|
+
subject(:validator) { described_class.new }
|
5
|
+
|
6
|
+
define :be_a_validator_error do # |expected|
|
7
|
+
match { false } # { |actual| some_condition }
|
8
|
+
end
|
9
|
+
|
10
|
+
describe '#validate' do
|
11
|
+
context 'without a parser' do
|
12
|
+
it 'raises ArgumentError' do
|
13
|
+
expect { validator.validate }.to raise_error ArgumentError
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'gives an empty error collection for a valid parser' do
|
18
|
+
expect(validator.validate(parser: valid_parser)).to be_empty if
|
19
|
+
defined?(valid_parser)
|
20
|
+
end
|
21
|
+
|
22
|
+
context 'for an invalid parser' do
|
23
|
+
it 'gives an non-empty error collection' do
|
24
|
+
expect(validator.validate(parser: invalid_parser)).not_to be_empty if
|
25
|
+
defined?(invalid_parser)
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'gives usable errors' do
|
29
|
+
pending 'we need to clarify the error type and usage'
|
30
|
+
|
31
|
+
validator.validate(parser: invalid_parser).each do |error|
|
32
|
+
expect(error).to be_a_validator_error
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
metadata
ADDED
@@ -0,0 +1,195 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: darlingtonia
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Tom Johnson
|
8
|
+
- Data Curation Experts
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2018-01-11 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: active-fedora
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - ">="
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '11.0'
|
21
|
+
- - "<="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: '12.99'
|
24
|
+
type: :runtime
|
25
|
+
prerelease: false
|
26
|
+
version_requirements: !ruby/object:Gem::Requirement
|
27
|
+
requirements:
|
28
|
+
- - ">="
|
29
|
+
- !ruby/object:Gem::Version
|
30
|
+
version: '11.0'
|
31
|
+
- - "<="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '12.99'
|
34
|
+
- !ruby/object:Gem::Dependency
|
35
|
+
name: yard
|
36
|
+
requirement: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0.9'
|
41
|
+
type: :development
|
42
|
+
prerelease: false
|
43
|
+
version_requirements: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0.9'
|
48
|
+
- !ruby/object:Gem::Dependency
|
49
|
+
name: bixby
|
50
|
+
requirement: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0.3'
|
55
|
+
type: :development
|
56
|
+
prerelease: false
|
57
|
+
version_requirements: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0.3'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: hyrax-spec
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0.2'
|
69
|
+
type: :development
|
70
|
+
prerelease: false
|
71
|
+
version_requirements: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0.2'
|
76
|
+
- !ruby/object:Gem::Dependency
|
77
|
+
name: rspec
|
78
|
+
requirement: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '3.6'
|
83
|
+
type: :development
|
84
|
+
prerelease: false
|
85
|
+
version_requirements: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '3.6'
|
90
|
+
- !ruby/object:Gem::Dependency
|
91
|
+
name: coveralls
|
92
|
+
requirement: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0.8'
|
97
|
+
type: :development
|
98
|
+
prerelease: false
|
99
|
+
version_requirements: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0.8'
|
104
|
+
- !ruby/object:Gem::Dependency
|
105
|
+
name: solr_wrapper
|
106
|
+
requirement: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0.3'
|
111
|
+
type: :development
|
112
|
+
prerelease: false
|
113
|
+
version_requirements: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0.3'
|
118
|
+
- !ruby/object:Gem::Dependency
|
119
|
+
name: fcrepo_wrapper
|
120
|
+
requirement: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0.9'
|
125
|
+
type: :development
|
126
|
+
prerelease: false
|
127
|
+
version_requirements: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - "~>"
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0.9'
|
132
|
+
description:
|
133
|
+
email: tom@curationexperts.com
|
134
|
+
executables: []
|
135
|
+
extensions: []
|
136
|
+
extra_rdoc_files: []
|
137
|
+
files:
|
138
|
+
- ".gitignore"
|
139
|
+
- ".rubocop.yml"
|
140
|
+
- ".travis.yml"
|
141
|
+
- Gemfile
|
142
|
+
- README.md
|
143
|
+
- Rakefile
|
144
|
+
- darlingtonia.gemspec
|
145
|
+
- lib/darlingtonia.rb
|
146
|
+
- lib/darlingtonia/always_invalid_validator.rb
|
147
|
+
- lib/darlingtonia/hash_mapper.rb
|
148
|
+
- lib/darlingtonia/importer.rb
|
149
|
+
- lib/darlingtonia/input_record.rb
|
150
|
+
- lib/darlingtonia/parser.rb
|
151
|
+
- lib/darlingtonia/parsers/csv_parser.rb
|
152
|
+
- lib/darlingtonia/record_importer.rb
|
153
|
+
- lib/darlingtonia/validator.rb
|
154
|
+
- lib/darlingtonia/validators/csv_format_validator.rb
|
155
|
+
- lib/darlingtonia/version.rb
|
156
|
+
- spec/darlingtonia/csv_format_validator_spec.rb
|
157
|
+
- spec/darlingtonia/csv_parser_spec.rb
|
158
|
+
- spec/darlingtonia/hash_mapper_spec.rb
|
159
|
+
- spec/darlingtonia/importer_spec.rb
|
160
|
+
- spec/darlingtonia/input_record_spec.rb
|
161
|
+
- spec/darlingtonia/parser_spec.rb
|
162
|
+
- spec/darlingtonia/validator_spec.rb
|
163
|
+
- spec/darlingtonia/version_spec.rb
|
164
|
+
- spec/fixtures/example.csv
|
165
|
+
- spec/integration/import_csv_spec.rb
|
166
|
+
- spec/spec_helper.rb
|
167
|
+
- spec/support/fakes/fake_parser.rb
|
168
|
+
- spec/support/shared_examples/a_mapper.rb
|
169
|
+
- spec/support/shared_examples/a_parser.rb
|
170
|
+
- spec/support/shared_examples/a_validator.rb
|
171
|
+
homepage:
|
172
|
+
licenses:
|
173
|
+
- Apache-2.0
|
174
|
+
metadata: {}
|
175
|
+
post_install_message:
|
176
|
+
rdoc_options: []
|
177
|
+
require_paths:
|
178
|
+
- lib
|
179
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
180
|
+
requirements:
|
181
|
+
- - ">="
|
182
|
+
- !ruby/object:Gem::Version
|
183
|
+
version: 2.3.4
|
184
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
185
|
+
requirements:
|
186
|
+
- - ">="
|
187
|
+
- !ruby/object:Gem::Version
|
188
|
+
version: '0'
|
189
|
+
requirements: []
|
190
|
+
rubyforge_project:
|
191
|
+
rubygems_version: 2.6.13
|
192
|
+
signing_key:
|
193
|
+
specification_version: 4
|
194
|
+
summary: Hyrax importers.
|
195
|
+
test_files: []
|