darlingtonia 2.0.0 → 2.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +9 -0
- data/.rubocop.yml +7 -0
- data/CHANGELOG.md +10 -1
- data/darlingtonia.gemspec +2 -1
- data/lib/darlingtonia.rb +3 -2
- data/lib/darlingtonia/hyrax_basic_metadata_mapper.rb +57 -13
- data/lib/darlingtonia/hyrax_record_importer.rb +83 -31
- data/lib/darlingtonia/log_stream.rb +43 -0
- data/lib/darlingtonia/spec/fakes/fake_parser.rb +0 -2
- data/lib/darlingtonia/version.rb +1 -1
- data/spec/darlingtonia/hyrax_basic_metadata_mapper_spec.rb +106 -2
- data/spec/darlingtonia/hyrax_record_importer_spec.rb +81 -29
- data/spec/darlingtonia/importer_spec.rb +3 -0
- data/spec/darlingtonia/input_record_spec.rb +2 -0
- data/spec/darlingtonia_spec.rb +4 -10
- data/spec/fixtures/hyrax/example.csv +3 -3
- data/spec/integration/import_hyrax_csv.rb +4 -10
- data/spec/support/shared_contexts/with_work_type.rb +31 -7
- metadata +19 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 145bf822e40f94c82cf2a0c61213e964a8f5e780209703e9506fa6ce35466612
|
4
|
+
data.tar.gz: ed804197a3ccad1047da9b8c8127603e38814c03d799325bb9c317938de71c88
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dbaec8b01d54cb94859e1630fec14b84082c981e25143646eb1aa58ab4dbb5e0bbebcb4954457799dccdac0d8cc20f78b315486f2f2d38bfa709bf209df70980
|
7
|
+
data.tar.gz: d5ce22c9e01ba293513d7398fa5af18bb5cfcfe1df67dc6313ebb70cce13c655dd8c1b0998042a1b823014337b2e2ae7980b288f5168a0d4d4caa5077d32701e
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
@@ -19,6 +19,10 @@ Metrics/BlockLength:
|
|
19
19
|
- 'spec/**/*'
|
20
20
|
- 'lib/darlingtonia/spec/**/*'
|
21
21
|
|
22
|
+
Metrics/CyclomaticComplexity:
|
23
|
+
Exclude:
|
24
|
+
- lib/darlingtonia/hyrax_basic_metadata_mapper.rb
|
25
|
+
|
22
26
|
Metrics/LineLength:
|
23
27
|
Exclude:
|
24
28
|
- 'lib/darlingtonia/hyrax_basic_metadata_mapper.rb'
|
@@ -47,3 +51,6 @@ RSpec/MultipleExpectations:
|
|
47
51
|
Exclude:
|
48
52
|
- 'spec/darlingtonia/hyrax_basic_metadata_mapper_spec.rb'
|
49
53
|
- 'spec/integration/import_hyrax_csv.rb'
|
54
|
+
|
55
|
+
Style/StructInheritance:
|
56
|
+
Enabled: false
|
data/CHANGELOG.md
CHANGED
@@ -1,9 +1,18 @@
|
|
1
|
+
2.1.0 - Tue Feb 12, 2019
|
2
|
+
|
3
|
+
* Map a variety of values for 'visibility' to their Hyrax approved equivalents
|
4
|
+
* Remove date_uploaded from HyraxBasicMetadataMapper so it doesn't over-write the Hyrax provided timestamp
|
5
|
+
* Import headers are now case and whitespace insensitive
|
6
|
+
* Raise a more meaningful error when an expected file isn't found
|
7
|
+
* Transform location URIs into a based_near hash so geonames URIs import properly into Hyrax
|
8
|
+
* Add a default LogStream class so that Darlingtonia has logging by default instead of only sending output to STDOUT
|
9
|
+
|
1
10
|
2.0.0 - Mon Feb 4, 2019
|
2
11
|
|
3
12
|
Assume our base use case is to import into Hyrax.
|
4
13
|
* Use HyraxRecordImporter as default
|
5
14
|
* Use HyraxMapper as default
|
6
|
-
* Add a getting started guide to the docs directory
|
15
|
+
* Add a getting started guide to the docs directory
|
7
16
|
|
8
17
|
1.2.3 - Thu Jan 24, 2019
|
9
18
|
------------------------
|
data/darlingtonia.gemspec
CHANGED
@@ -20,13 +20,14 @@ Gem::Specification.new do |gem|
|
|
20
20
|
gem.add_dependency 'active-fedora', '>= 11.5.2'
|
21
21
|
|
22
22
|
gem.add_development_dependency 'yard', '~> 0.9'
|
23
|
-
gem.add_development_dependency 'bixby', '~> 0
|
23
|
+
gem.add_development_dependency 'bixby', '~> 1.0'
|
24
24
|
gem.add_development_dependency 'hyrax-spec', '~> 0.2'
|
25
25
|
gem.add_development_dependency 'rspec', '~> 3.6'
|
26
26
|
gem.add_development_dependency 'coveralls', '~> 0.8'
|
27
27
|
gem.add_development_dependency 'solr_wrapper', '~> 2.1'
|
28
28
|
gem.add_development_dependency 'fcrepo_wrapper', '~> 0.9'
|
29
29
|
gem.add_development_dependency 'byebug'
|
30
|
+
gem.add_development_dependency 'rake'
|
30
31
|
|
31
32
|
gem.files = `git ls-files`.split("\n")
|
32
33
|
gem.test_files = `git ls-files -- {spec}/*`.split("\n")
|
data/lib/darlingtonia.rb
CHANGED
@@ -33,6 +33,7 @@ module Darlingtonia
|
|
33
33
|
end
|
34
34
|
module_function :config
|
35
35
|
|
36
|
+
require 'darlingtonia/log_stream'
|
36
37
|
##
|
37
38
|
# Module-wide options for `Darlingtonia`.
|
38
39
|
class Configuration
|
@@ -44,8 +45,8 @@ module Darlingtonia
|
|
44
45
|
attr_accessor :default_error_stream, :default_info_stream
|
45
46
|
|
46
47
|
def initialize
|
47
|
-
self.default_error_stream =
|
48
|
-
self.default_info_stream =
|
48
|
+
self.default_error_stream = Darlingtonia::LogStream.new
|
49
|
+
self.default_info_stream = Darlingtonia::LogStream.new
|
49
50
|
end
|
50
51
|
end
|
51
52
|
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
require 'uri'
|
2
3
|
|
3
4
|
module Darlingtonia
|
4
5
|
##
|
@@ -37,31 +38,52 @@ module Darlingtonia
|
|
37
38
|
# Hyrax should return a single value instead of
|
38
39
|
# an Array of values.
|
39
40
|
def depositor
|
40
|
-
|
41
|
-
end
|
42
|
-
|
43
|
-
def date_uploaded
|
44
|
-
metadata['date_uploaded']
|
41
|
+
single_value('depositor')
|
45
42
|
end
|
46
43
|
|
47
44
|
def date_modified
|
48
|
-
|
45
|
+
single_value('date_modified')
|
49
46
|
end
|
50
47
|
|
51
48
|
def label
|
52
|
-
|
49
|
+
single_value('label')
|
53
50
|
end
|
54
51
|
|
55
52
|
def relative_path
|
56
|
-
|
53
|
+
single_value('relative_path')
|
57
54
|
end
|
58
55
|
|
59
56
|
def import_url
|
60
|
-
|
57
|
+
single_value('import_url')
|
61
58
|
end
|
62
59
|
|
60
|
+
# We should accept visibility values that match the UI and transform them into
|
61
|
+
# the controlled vocabulary term expected by Hyrax
|
63
62
|
def visibility
|
64
|
-
metadata['visibility']
|
63
|
+
case metadata[matching_header('visibility')]&.downcase&.gsub(/\s+/, "")
|
64
|
+
when 'public'
|
65
|
+
'open'
|
66
|
+
when 'open'
|
67
|
+
'open'
|
68
|
+
when 'registered'
|
69
|
+
'authenticated'
|
70
|
+
when "authenticated"
|
71
|
+
'authenticated'
|
72
|
+
when ::Hyrax::Institution.name&.downcase&.gsub(/\s+/, "")
|
73
|
+
'authenticated'
|
74
|
+
when ::Hyrax::Institution.name_full&.downcase&.gsub(/\s+/, "")
|
75
|
+
'authenticated'
|
76
|
+
when 'private'
|
77
|
+
'restricted'
|
78
|
+
when 'restricted'
|
79
|
+
'restricted'
|
80
|
+
else
|
81
|
+
'restricted' # This is the default if nothing else matches
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def files
|
86
|
+
map_field('files')
|
65
87
|
end
|
66
88
|
|
67
89
|
##
|
@@ -79,16 +101,38 @@ module Darlingtonia
|
|
79
101
|
##
|
80
102
|
# @see MetadataMapper#map_field
|
81
103
|
def map_field(name)
|
82
|
-
method_name = name
|
104
|
+
method_name = name.to_s
|
83
105
|
method_name = CSV_HEADERS[name] if CSV_HEADERS.keys.include?(name)
|
84
|
-
|
106
|
+
key = matching_header(method_name)
|
107
|
+
Array(metadata[key]&.split(delimiter))
|
85
108
|
end
|
86
109
|
|
87
110
|
protected
|
88
111
|
|
112
|
+
# Some fields should have single values instead
|
113
|
+
# of array values.
|
114
|
+
def single_value(field_name)
|
115
|
+
metadata[matching_header(field_name)]
|
116
|
+
end
|
117
|
+
|
118
|
+
# Lenient matching for headers.
|
119
|
+
# If the user has headers like:
|
120
|
+
# 'Title' or 'TITLE' or 'Title '
|
121
|
+
# it should match the :title field.
|
122
|
+
def matching_header(field_name)
|
123
|
+
metadata.keys.find do |key|
|
124
|
+
next unless key
|
125
|
+
key.downcase.strip == field_name
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
89
129
|
# Properties defined in Hyrax::CoreMetadata
|
130
|
+
# Note that date_uploaded is NOT set here, even though it is defined in
|
131
|
+
# Hyrax::CoreMetadata. Hyrax expects to set date_uploaded itself, and
|
132
|
+
# sending a metadata value for that field interferes with Hyrax expected
|
133
|
+
# behavior.
|
90
134
|
def core_fields
|
91
|
-
[:depositor, :title, :
|
135
|
+
[:depositor, :title, :date_modified]
|
92
136
|
end
|
93
137
|
|
94
138
|
# Properties defined in Hyrax::BasicMetadata
|
@@ -1,26 +1,42 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module Darlingtonia
|
3
4
|
class HyraxRecordImporter < RecordImporter
|
4
5
|
# TODO: Get this from Hyrax config
|
5
6
|
DEFAULT_CREATOR_KEY = 'batchuser@example.com'
|
6
7
|
|
7
|
-
|
8
|
-
#
|
9
|
-
|
10
|
-
|
8
|
+
# @!attribute [rw] depositor
|
9
|
+
# @return [User]
|
10
|
+
attr_accessor :depositor
|
11
|
+
|
12
|
+
# @!attribute [rw] collection_id
|
13
|
+
# @return [String] The fedora ID for a Collection.
|
11
14
|
attr_accessor :collection_id
|
12
15
|
|
13
|
-
|
14
|
-
#
|
16
|
+
# @param attributes [Hash] Attributes that come
|
17
|
+
# from the UI or importer rather than from
|
18
|
+
# the CSV/mapper.
|
19
|
+
# @example
|
20
|
+
# attributes: { collection_id: '123',
|
21
|
+
# depositor_id: '456' }
|
15
22
|
def initialize(error_stream: Darlingtonia.config.default_error_stream,
|
16
23
|
info_stream: Darlingtonia.config.default_info_stream,
|
17
|
-
|
18
|
-
self.
|
19
|
-
|
24
|
+
attributes: {})
|
25
|
+
self.collection_id = attributes[:collection_id]
|
26
|
+
set_depositor(attributes[:depositor_id])
|
20
27
|
|
21
28
|
super(error_stream: error_stream, info_stream: info_stream)
|
22
29
|
end
|
23
30
|
|
31
|
+
# "depositor" is a required field for Hyrax. If
|
32
|
+
# it hasn't been set, set it to the Hyrax default
|
33
|
+
# batch user.
|
34
|
+
def set_depositor(user_id)
|
35
|
+
user = User.find(user_id) if user_id
|
36
|
+
user ||= ::User.find_or_create_system_user(DEFAULT_CREATOR_KEY)
|
37
|
+
self.depositor = user
|
38
|
+
end
|
39
|
+
|
24
40
|
##
|
25
41
|
# @param record [ImportRecord]
|
26
42
|
#
|
@@ -42,14 +58,6 @@ module Darlingtonia
|
|
42
58
|
Hyrax.config.curation_concerns.first
|
43
59
|
end
|
44
60
|
|
45
|
-
# Depositor is a required field for Hyrax
|
46
|
-
# If it hasn't been set in the metadata, set it to the Hyrax default batch user
|
47
|
-
# Even if it has been set, for now, override that with the batch user.
|
48
|
-
# @param Darlingtonia::InputRecord
|
49
|
-
def set_depositor(record)
|
50
|
-
record.mapper.metadata["depositor"] = @creator.user_key
|
51
|
-
end
|
52
|
-
|
53
61
|
# The path on disk where file attachments can be found
|
54
62
|
def file_attachments_path
|
55
63
|
ENV['IMPORT_PATH'] || '/opt/data'
|
@@ -61,15 +69,13 @@ module Darlingtonia
|
|
61
69
|
# @param [Darlingtonia::InputRecord]
|
62
70
|
# @return [Array] an array of Hyrax::UploadedFile ids
|
63
71
|
def create_upload_files(record)
|
64
|
-
|
65
|
-
return [] if
|
66
|
-
|
67
|
-
files_to_attach = file_attachment_filenames.split(record.mapper.delimiter)
|
72
|
+
files_to_attach = record.mapper.files
|
73
|
+
return [] if files_to_attach.nil? || files_to_attach.empty?
|
68
74
|
|
69
75
|
uploaded_file_ids = []
|
70
76
|
files_to_attach.each do |filename|
|
71
77
|
file = File.open(find_file_path(filename))
|
72
|
-
uploaded_file = Hyrax::UploadedFile.create(user: @
|
78
|
+
uploaded_file = Hyrax::UploadedFile.create(user: @depositor, file: file)
|
73
79
|
uploaded_file_ids << uploaded_file.id
|
74
80
|
file.close
|
75
81
|
end
|
@@ -79,10 +85,51 @@ module Darlingtonia
|
|
79
85
|
##
|
80
86
|
# Within the directory specified by ENV['IMPORT_PATH'], find the first
|
81
87
|
# instance of a file matching the given filename.
|
88
|
+
# If there is no matching file, raise an exception.
|
82
89
|
# @param [String] filename
|
83
90
|
# @return [String] a full pathname to the found file
|
84
91
|
def find_file_path(filename)
|
85
|
-
Dir.glob("#{ENV['IMPORT_PATH']}/**/#{filename}").first
|
92
|
+
filepath = Dir.glob("#{ENV['IMPORT_PATH']}/**/#{filename}").first
|
93
|
+
raise "Cannot find file #{filename}... Are you sure it has been uploaded and that the filename matches?" if filepath.nil?
|
94
|
+
filepath
|
95
|
+
end
|
96
|
+
|
97
|
+
##
|
98
|
+
# When submitting location data (a.k.a. the "based near" attribute) via the UI,
|
99
|
+
# Hyrax expects to receive a `based_near_attributes` hash in a specific format.
|
100
|
+
# We need to take geonames urls as provided by the customer and transform them to
|
101
|
+
# mimic what the Hyrax UI would ordinarily produce. These will get turned into
|
102
|
+
# Hyrax::ControlledVocabularies::Location objects upon ingest.
|
103
|
+
# The expected hash looks like this:
|
104
|
+
# "based_near_attributes"=>
|
105
|
+
# {
|
106
|
+
# "0"=> {
|
107
|
+
# "id"=>"http://sws.geonames.org/5667009/", "_destroy"=>""
|
108
|
+
# },
|
109
|
+
# "1"=> {
|
110
|
+
# "id"=>"http://sws.geonames.org/6252001/", "_destroy"=>""
|
111
|
+
# },
|
112
|
+
# }
|
113
|
+
# @return [Hash] a "based_near_attributes" hash as
|
114
|
+
def based_near_attributes(based_near)
|
115
|
+
original_geonames_uris = based_near
|
116
|
+
return if original_geonames_uris.empty?
|
117
|
+
based_near_attributes = {}
|
118
|
+
original_geonames_uris.each_with_index do |uri, i|
|
119
|
+
based_near_attributes[i.to_s] = { 'id' => uri_to_sws(uri), "_destroy" => "" }
|
120
|
+
end
|
121
|
+
based_near_attributes
|
122
|
+
end
|
123
|
+
|
124
|
+
#
|
125
|
+
# Take a user-facing geonames URI and return an sws URI, of the form Hyrax expects
|
126
|
+
# (e.g., "http://sws.geonames.org/6252001/")
|
127
|
+
# @param [String] uri
|
128
|
+
# @return [String] an sws style geonames uri
|
129
|
+
def uri_to_sws(uri)
|
130
|
+
uri = URI(uri)
|
131
|
+
geonames_number = uri.path.split('/')[1]
|
132
|
+
"http://sws.geonames.org/#{geonames_number}/"
|
86
133
|
end
|
87
134
|
|
88
135
|
private
|
@@ -92,16 +139,21 @@ module Darlingtonia
|
|
92
139
|
def create_for(record:)
|
93
140
|
info_stream << 'Creating record: ' \
|
94
141
|
"#{record.respond_to?(:title) ? record.title : record}."
|
95
|
-
|
96
|
-
|
97
|
-
|
142
|
+
additional_attrs = {
|
143
|
+
uploaded_files: create_upload_files(record),
|
144
|
+
depositor: @depositor.user_key
|
145
|
+
}
|
146
|
+
created = import_type.new
|
147
|
+
|
148
|
+
attrs = record.attributes.merge(additional_attrs)
|
149
|
+
attrs = attrs.merge(member_of_collections_attributes: { '0' => { id: collection_id } }) if collection_id
|
98
150
|
|
99
|
-
|
100
|
-
|
151
|
+
based_near = attrs.delete(:based_near)
|
152
|
+
attrs = attrs.merge(based_near_attributes: based_near_attributes(based_near)) unless based_near.nil? || based_near.empty?
|
101
153
|
|
102
|
-
actor_env
|
103
|
-
|
104
|
-
|
154
|
+
actor_env = Hyrax::Actors::Environment.new(created,
|
155
|
+
::Ability.new(@depositor),
|
156
|
+
attrs)
|
105
157
|
|
106
158
|
if Hyrax::CurationConcern.actor.create(actor_env)
|
107
159
|
info_stream << "Record created at: #{created.id}"
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Darlingtonia
|
4
|
+
class LogStream
|
5
|
+
##
|
6
|
+
# @!attribute [rw] logger
|
7
|
+
# @return [Logger]
|
8
|
+
# @!attribute [rw] severity
|
9
|
+
# @return [Logger::Serverity]
|
10
|
+
attr_accessor :logger, :severity
|
11
|
+
|
12
|
+
def initialize(logger: nil, severity: nil)
|
13
|
+
self.logger = logger || Logger.new(build_filename)
|
14
|
+
self.severity = severity || Logger::INFO
|
15
|
+
end
|
16
|
+
|
17
|
+
def <<(msg)
|
18
|
+
logger.add(severity, msg)
|
19
|
+
STDOUT << msg
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def build_filename
|
25
|
+
return ENV['IMPORT_LOG'] if ENV['IMPORT_LOG']
|
26
|
+
return rails_log_name if rails_log_name
|
27
|
+
return './darlingtonia_import.log' unless rails_log_name
|
28
|
+
end
|
29
|
+
|
30
|
+
def rails_log_name
|
31
|
+
case Rails.env
|
32
|
+
when 'production'
|
33
|
+
Rails.root.join('log', "csv_import.log").to_s
|
34
|
+
when 'development'
|
35
|
+
Rails.root.join('log', "dev_csv_import.log").to_s
|
36
|
+
when 'test'
|
37
|
+
Rails.root.join('log', "test_csv_import.log").to_s
|
38
|
+
end
|
39
|
+
rescue NameError
|
40
|
+
false
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -14,11 +14,9 @@ class FakeParser < Darlingtonia::Parser
|
|
14
14
|
end
|
15
15
|
end
|
16
16
|
|
17
|
-
# rubocop:disable RSpec/FilePath
|
18
17
|
describe FakeParser do
|
19
18
|
it_behaves_like 'a Darlingtonia::Parser' do
|
20
19
|
subject(:parser) { described_class.new }
|
21
20
|
let(:record_count) { 3 }
|
22
21
|
end
|
23
22
|
end
|
24
|
-
# rubocop:enable RSpec/FilePath
|
data/lib/darlingtonia/version.rb
CHANGED
@@ -6,7 +6,7 @@ describe Darlingtonia::HyraxBasicMetadataMapper do
|
|
6
6
|
|
7
7
|
# Properties defined in Hyrax::CoreMetadata
|
8
8
|
let(:core_fields) do
|
9
|
-
[:depositor, :title, :
|
9
|
+
[:depositor, :title, :date_modified]
|
10
10
|
end
|
11
11
|
|
12
12
|
# Properties defined in Hyrax::BasicMetadata
|
@@ -49,7 +49,6 @@ describe Darlingtonia::HyraxBasicMetadataMapper do
|
|
49
49
|
|
50
50
|
it 'returns single values for single-value fields' do
|
51
51
|
expect(mapper.depositor).to eq 'someone@example.org'
|
52
|
-
expect(mapper.date_uploaded).to eq nil
|
53
52
|
expect(mapper.date_modified).to eq nil
|
54
53
|
expect(mapper.label).to eq nil
|
55
54
|
expect(mapper.relative_path).to eq nil
|
@@ -83,4 +82,109 @@ describe Darlingtonia::HyraxBasicMetadataMapper do
|
|
83
82
|
expect(mapper.delimiter).to eq 'ಠ_ಠ'
|
84
83
|
end
|
85
84
|
end
|
85
|
+
|
86
|
+
describe 'lenient headers' do
|
87
|
+
context 'headers with capital letters' do
|
88
|
+
before { mapper.metadata = metadata }
|
89
|
+
let(:metadata) do
|
90
|
+
{ 'Title' => 'A Title',
|
91
|
+
'Related URL' => 'http://example.com',
|
92
|
+
'Abstract or Summary' => 'desc1|~|desc2',
|
93
|
+
'visiBILITY' => 'open',
|
94
|
+
'Depositor' => 'someone@example.org',
|
95
|
+
'DATE_modified' => 'mod date',
|
96
|
+
'laBel' => 'label',
|
97
|
+
'relative_PATH' => 'rel path',
|
98
|
+
'import_URL' => 'imp url' }
|
99
|
+
end
|
100
|
+
|
101
|
+
it 'matches the correct fields' do
|
102
|
+
expect(mapper.title).to eq ['A Title']
|
103
|
+
expect(mapper.related_url).to eq ['http://example.com']
|
104
|
+
expect(mapper.description).to eq ['desc1', 'desc2']
|
105
|
+
expect(mapper.creator).to eq []
|
106
|
+
expect(mapper.visibility).to eq 'open'
|
107
|
+
expect(mapper.depositor).to eq 'someone@example.org'
|
108
|
+
expect(mapper.date_modified).to eq 'mod date'
|
109
|
+
expect(mapper.label).to eq 'label'
|
110
|
+
expect(mapper.relative_path).to eq 'rel path'
|
111
|
+
expect(mapper.import_url).to eq 'imp url'
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
context 'headers with sloppy whitespace' do
|
116
|
+
before { mapper.metadata = metadata }
|
117
|
+
let(:metadata) do
|
118
|
+
{ ' Title ' => 'A Title',
|
119
|
+
" Related URL \n " => 'http://example.com',
|
120
|
+
' visiBILITY ' => 'open' }
|
121
|
+
end
|
122
|
+
|
123
|
+
it 'matches the correct fields' do
|
124
|
+
expect(mapper.title).to eq ['A Title']
|
125
|
+
expect(mapper.related_url).to eq ['http://example.com']
|
126
|
+
expect(mapper.visibility).to eq 'open'
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
context 'Visibility values in the CSV should match the Edit UI' do
|
131
|
+
load File.expand_path("../../support/shared_contexts/with_work_type.rb", __FILE__)
|
132
|
+
include_context 'with a work type'
|
133
|
+
context 'public is a synonym for open' do
|
134
|
+
before { mapper.metadata = metadata }
|
135
|
+
let(:metadata) do
|
136
|
+
{ ' Title ' => 'A Title',
|
137
|
+
" Related URL \n " => 'http://example.com',
|
138
|
+
' visiBILITY ' => 'PubLIC' }
|
139
|
+
end
|
140
|
+
|
141
|
+
it 'transforms public to open regardless of capitalization' do
|
142
|
+
expect(mapper.title).to eq ['A Title']
|
143
|
+
expect(mapper.related_url).to eq ['http://example.com']
|
144
|
+
expect(mapper.visibility).to eq 'open'
|
145
|
+
end
|
146
|
+
end
|
147
|
+
context 'institution name is a synonym for authenticated' do
|
148
|
+
before { mapper.metadata = metadata }
|
149
|
+
let(:metadata) do
|
150
|
+
{ ' Title ' => 'A Title',
|
151
|
+
" Related URL \n " => 'http://example.com',
|
152
|
+
' visiBILITY ' => 'my_institution' }
|
153
|
+
end
|
154
|
+
|
155
|
+
it 'transforms institution name to authenticated regardless of capitalization' do
|
156
|
+
expect(mapper.title).to eq ['A Title']
|
157
|
+
expect(mapper.related_url).to eq ['http://example.com']
|
158
|
+
expect(mapper.visibility).to eq 'authenticated'
|
159
|
+
end
|
160
|
+
end
|
161
|
+
context 'full institution name is a synonym for authenticated' do
|
162
|
+
before { mapper.metadata = metadata }
|
163
|
+
let(:metadata) do
|
164
|
+
{ ' Title ' => 'A Title',
|
165
|
+
" Related URL \n " => 'http://example.com',
|
166
|
+
' visiBILITY ' => 'my full institution name' }
|
167
|
+
end
|
168
|
+
|
169
|
+
it 'transforms full institution name to authenticated regardless of capitalization' do
|
170
|
+
expect(mapper.title).to eq ['A Title']
|
171
|
+
expect(mapper.related_url).to eq ['http://example.com']
|
172
|
+
expect(mapper.visibility).to eq 'authenticated'
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
# When someone accidentally has too many commas in the CSV rows
|
178
|
+
context 'headers with a nil' do
|
179
|
+
before { mapper.metadata = metadata }
|
180
|
+
let(:metadata) do
|
181
|
+
{ ' Title ' => 'A Title',
|
182
|
+
nil => nil }
|
183
|
+
end
|
184
|
+
|
185
|
+
it 'doesn\'t raise an error for missing fields' do
|
186
|
+
expect(mapper.depositor).to eq nil
|
187
|
+
end
|
188
|
+
end
|
189
|
+
end
|
86
190
|
end
|
@@ -6,19 +6,19 @@ describe Darlingtonia::HyraxRecordImporter, :clean do
|
|
6
6
|
described_class.new(error_stream: error_stream, info_stream: info_stream)
|
7
7
|
end
|
8
8
|
|
9
|
+
load File.expand_path("../../support/shared_contexts/with_work_type.rb", __FILE__)
|
10
|
+
include_context 'with a work type'
|
11
|
+
|
9
12
|
let(:error_stream) { [] }
|
10
13
|
let(:info_stream) { [] }
|
11
14
|
let(:record) { Darlingtonia::InputRecord.from(metadata: metadata) }
|
12
15
|
|
13
16
|
context 'collection id' do
|
14
17
|
subject(:importer) do
|
15
|
-
described_class.new(collection_id: collection_id)
|
18
|
+
described_class.new(attributes: { collection_id: collection_id })
|
16
19
|
end
|
17
20
|
let(:collection_id) { '123' }
|
18
21
|
|
19
|
-
load File.expand_path("../../support/shared_contexts/with_work_type.rb", __FILE__)
|
20
|
-
include_context 'with a work type'
|
21
|
-
|
22
22
|
it 'can have a collection id' do
|
23
23
|
expect(importer.collection_id).to eq collection_id
|
24
24
|
end
|
@@ -32,8 +32,6 @@ describe Darlingtonia::HyraxRecordImporter, :clean do
|
|
32
32
|
'visibility' => 'open'
|
33
33
|
}
|
34
34
|
end
|
35
|
-
load File.expand_path("../../support/shared_contexts/with_work_type.rb", __FILE__)
|
36
|
-
include_context 'with a work type'
|
37
35
|
|
38
36
|
it 'creates a work for record' do
|
39
37
|
expect { importer.import(record: record) }
|
@@ -54,8 +52,6 @@ describe Darlingtonia::HyraxRecordImporter, :clean do
|
|
54
52
|
'files' => 'darlingtonia.png|~|cat.png'
|
55
53
|
}
|
56
54
|
end
|
57
|
-
load File.expand_path("../../support/shared_contexts/with_work_type.rb", __FILE__)
|
58
|
-
include_context 'with a work type'
|
59
55
|
it 'finds a file even if it is in a subdirectory' do
|
60
56
|
expect(importer.find_file_path('cat.png')).to eq "#{ENV['IMPORT_PATH']}/animals/cat.png"
|
61
57
|
end
|
@@ -71,33 +67,89 @@ describe Darlingtonia::HyraxRecordImporter, :clean do
|
|
71
67
|
end
|
72
68
|
end
|
73
69
|
|
74
|
-
context 'with and
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
70
|
+
context 'with attached files, alternate capitalization and whitespace in "files" header' do
|
71
|
+
before do
|
72
|
+
ENV['IMPORT_PATH'] = File.expand_path('../fixtures/images', File.dirname(__FILE__))
|
73
|
+
end
|
74
|
+
let(:metadata) do
|
75
|
+
{
|
76
|
+
'title' => 'A Title',
|
77
|
+
'visibility' => 'open',
|
78
|
+
' Files' => 'darlingtonia.png|~|cat.png'
|
79
|
+
}
|
80
|
+
end
|
81
|
+
|
82
|
+
it 'makes an uploaded file object for each file attachment' do
|
83
|
+
expect { importer.import(record: record) }
|
84
|
+
.to change { Hyrax::UploadedFile.count }
|
85
|
+
.by 2
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
context 'with missing files' do
|
90
|
+
before do
|
91
|
+
ENV['IMPORT_PATH'] = File.expand_path('../fixtures/images', File.dirname(__FILE__))
|
92
|
+
end
|
93
|
+
it 'raises an exception' do
|
94
|
+
expect { importer.find_file_path('foo.png') }.to raise_exception(RuntimeError)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
describe '#set_depositor' do
|
99
|
+
let(:metadata) { { 'title' => 'A Title' } }
|
100
|
+
|
101
|
+
context 'when no depositor is set' do
|
102
|
+
it 'sets the Hyrax default batch user' do
|
103
|
+
expect(importer.depositor.user_key).to eq 'batchuser@example.com'
|
82
104
|
end
|
83
|
-
|
84
|
-
|
85
|
-
|
105
|
+
end
|
106
|
+
|
107
|
+
context 'when depositor is passed to initializer' do
|
108
|
+
subject(:importer) { described_class.new(error_stream: error_stream, info_stream: info_stream, attributes: { depositor_id: user.id }) }
|
109
|
+
|
110
|
+
let(:user) { ::User.new(id: '123', user_key: 'special_user@example.com') }
|
111
|
+
before { allow(::User).to receive(:find).and_return(user) }
|
112
|
+
|
113
|
+
it 'sets it to the passed-in depositor' do
|
114
|
+
expect(importer.depositor.user_key).to eq 'special_user@example.com'
|
86
115
|
end
|
87
116
|
end
|
88
|
-
|
117
|
+
|
118
|
+
context 'when depositor is set in metadata' do
|
89
119
|
let(:metadata) do
|
90
|
-
{
|
91
|
-
'
|
92
|
-
'language' => 'English',
|
93
|
-
'visibility' => 'open',
|
94
|
-
'depositor' => 'my@custom.depositor'
|
95
|
-
}
|
120
|
+
{ 'title' => 'A Title',
|
121
|
+
'Depositor' => 'metadata_user@example.com' }
|
96
122
|
end
|
97
|
-
|
98
|
-
|
99
|
-
expect(
|
123
|
+
|
124
|
+
it 'sets the Hyrax default batch user' do
|
125
|
+
expect(importer.depositor.user_key).to eq 'batchuser@example.com'
|
126
|
+
# TODO: expect(importer.depositor.user_key).to eq 'metadata_user@example.com'
|
127
|
+
# The metadata depositor should probably override any passed-in or default depositor.
|
100
128
|
end
|
101
129
|
end
|
102
130
|
end
|
131
|
+
# When submitting location data (a.k.a., the "based near" attribute) via the UI,
|
132
|
+
# Hyrax expects to receive a `based_near_attributes` hash in a specific format.
|
133
|
+
# We need to take geonames urls as provided by the customer and transform them to
|
134
|
+
# mimic what the Hyrax UI would ordinarily produce. These will get turned into
|
135
|
+
# Hyrax::ControlledVocabularies::Location objects upon ingest.
|
136
|
+
context 'with location uris' do
|
137
|
+
let(:based_near) { ['http://www.geonames.org/5667009/montana.html', 'http://www.geonames.org/6252001/united-states.html'] }
|
138
|
+
let(:expected_bn_hash) do
|
139
|
+
{
|
140
|
+
"0" => {
|
141
|
+
"id" => "http://sws.geonames.org/5667009/", "_destroy" => ""
|
142
|
+
},
|
143
|
+
"1" => {
|
144
|
+
"id" => "http://sws.geonames.org/6252001/", "_destroy" => ""
|
145
|
+
}
|
146
|
+
}
|
147
|
+
end
|
148
|
+
it "gets a sws uri from a geonames uri" do
|
149
|
+
expect(importer.uri_to_sws("http://www.geonames.org/6252001/united-states.html")).to eq "http://sws.geonames.org/6252001/"
|
150
|
+
end
|
151
|
+
it 'transforms an array of geonames uris into the expected based_near_attributes hash' do
|
152
|
+
expect(importer.based_near_attributes(based_near)).to eq expected_bn_hash
|
153
|
+
end
|
154
|
+
end
|
103
155
|
end
|
@@ -3,6 +3,9 @@
|
|
3
3
|
require 'spec_helper'
|
4
4
|
|
5
5
|
describe Darlingtonia::Importer do
|
6
|
+
load File.expand_path("../../support/shared_contexts/with_work_type.rb", __FILE__)
|
7
|
+
include_context 'with a work type'
|
8
|
+
|
6
9
|
subject(:importer) { described_class.new(parser: parser) }
|
7
10
|
let(:parser) { FakeParser.new(file: input) }
|
8
11
|
let(:input) { [{ 'title' => '1' }, { 'title' => '2' }, { 'title' => '3' }] }
|
@@ -3,6 +3,8 @@
|
|
3
3
|
require 'spec_helper'
|
4
4
|
|
5
5
|
describe Darlingtonia::InputRecord do
|
6
|
+
load File.expand_path("../../support/shared_contexts/with_work_type.rb", __FILE__)
|
7
|
+
include_context 'with a work type'
|
6
8
|
subject(:record) { described_class.from(metadata: metadata) }
|
7
9
|
|
8
10
|
let(:metadata) do
|
data/spec/darlingtonia_spec.rb
CHANGED
@@ -5,21 +5,15 @@ require 'spec_helper'
|
|
5
5
|
describe Darlingtonia do
|
6
6
|
describe '#config' do
|
7
7
|
it 'can set a default error stream' do
|
8
|
-
|
9
|
-
|
10
|
-
expect { described_class.config { |c| c.default_error_stream = stream } }
|
8
|
+
expect { described_class.config { |c| c.default_error_stream = STDOUT } }
|
11
9
|
.to change { described_class.config.default_error_stream }
|
12
|
-
.
|
13
|
-
.to(stream)
|
10
|
+
.to(STDOUT)
|
14
11
|
end
|
15
12
|
|
16
13
|
it 'can set a default info stream' do
|
17
|
-
|
18
|
-
|
19
|
-
expect { described_class.config { |c| c.default_info_stream = stream } }
|
14
|
+
expect { described_class.config { |c| c.default_info_stream = STDOUT } }
|
20
15
|
.to change { described_class.config.default_info_stream }
|
21
|
-
.
|
22
|
-
.to(stream)
|
16
|
+
.to(STDOUT)
|
23
17
|
end
|
24
18
|
end
|
25
19
|
end
|
@@ -1,3 +1,3 @@
|
|
1
|
-
title,depositor,
|
2
|
-
Work 1 Title,user@example.com,2018-
|
3
|
-
Work 2 Title
|
1
|
+
title,depositor,date_modified,label,relative_path,import_url,resource type,creator,contributor,abstract or summary,keyword,license,rights statement,publisher,date created,subject,language,identifier,location,related url,bibliographic_citation,source
|
2
|
+
Work 1 Title,user@example.com,2018-01-01,Work 1 Label,tmp/files,https://example.com,Work 1 Type,Work 1 creator,Work 1 contrib,Desc 1,Key 1,Lic 1,RS 1,Pub 1,2018-06-06,Subj 1,English|~|Japanese,Ident 1,Based 1,https://example.com/related,Bib 1,Source 1
|
3
|
+
Work 2 Title,1970-12-21,,Work 2 Label,,,Work 2 Type,,,Desc 2,,,,Pub 2,,Subj 2
|
@@ -2,17 +2,13 @@
|
|
2
2
|
require 'spec_helper'
|
3
3
|
|
4
4
|
describe 'importing a CSV with Hyrax defaults', :clean do
|
5
|
-
subject(:importer) { Darlingtonia::Importer.new(parser: parser) }
|
5
|
+
subject(:importer) { Darlingtonia::Importer.new(parser: parser, record_importer: record_importer) }
|
6
6
|
let(:parser) { Darlingtonia::CsvParser.new(file: csv_file) }
|
7
|
+
let(:record_importer) { Darlingtonia::HyraxRecordImporter.new }
|
7
8
|
|
8
9
|
let(:csv_file) { File.open('spec/fixtures/hyrax/example.csv') }
|
9
10
|
after { csv_file.close }
|
10
11
|
|
11
|
-
before do
|
12
|
-
# Force it to use the Hyrax mapper instead of the default mapper
|
13
|
-
allow(Darlingtonia::HashMapper).to receive(:new).and_return(Darlingtonia::HyraxBasicMetadataMapper.new)
|
14
|
-
end
|
15
|
-
|
16
12
|
load File.expand_path("../../support/shared_contexts/with_work_type.rb", __FILE__)
|
17
13
|
include_context 'with a work type'
|
18
14
|
|
@@ -24,8 +20,7 @@ describe 'importing a CSV with Hyrax defaults', :clean do
|
|
24
20
|
work2 = works.find { |w| w.title == ['Work 2 Title'] }
|
25
21
|
|
26
22
|
# First Record
|
27
|
-
expect(work1.depositor).to eq '
|
28
|
-
expect(work1.date_uploaded).to eq '2018-12-21'
|
23
|
+
expect(work1.depositor).to eq 'batchuser@example.com'
|
29
24
|
expect(work1.date_modified).to eq '2018-01-01'
|
30
25
|
expect(work1.label).to eq 'Work 1 Label'
|
31
26
|
expect(work1.relative_path).to eq 'tmp/files'
|
@@ -51,8 +46,7 @@ describe 'importing a CSV with Hyrax defaults', :clean do
|
|
51
46
|
expect(work1.source).to eq ['Source 1']
|
52
47
|
|
53
48
|
# Second Record
|
54
|
-
expect(work2.depositor).to
|
55
|
-
expect(work2.date_uploaded).to eq '1970-12-21'
|
49
|
+
expect(work2.depositor).to eq 'batchuser@example.com'
|
56
50
|
expect(work2.date_modified).to be_nil
|
57
51
|
expect(work2.label).to eq 'Work 2 Label'
|
58
52
|
expect(work2.relative_path).to be_nil
|
@@ -8,16 +8,22 @@ shared_context 'with a work type' do
|
|
8
8
|
|
9
9
|
class Work < ActiveFedora::Base
|
10
10
|
attr_accessor :visibility
|
11
|
+
attr_accessor :based_near_attributes
|
11
12
|
include ::Hyrax::CoreMetadata
|
12
13
|
include ::Hyrax::BasicMetadata
|
13
14
|
end
|
14
15
|
|
15
|
-
class User
|
16
|
+
class User < Struct.new(:id, :user_key)
|
17
|
+
def initialize(inputs = {})
|
18
|
+
self.id = inputs[:id]
|
19
|
+
self.user_key = inputs[:user_key] || batch_user_key
|
20
|
+
end
|
21
|
+
|
16
22
|
def self.find_or_create_system_user(_email)
|
17
23
|
User.new
|
18
24
|
end
|
19
25
|
|
20
|
-
def
|
26
|
+
def batch_user_key
|
21
27
|
'batchuser@example.com'
|
22
28
|
end
|
23
29
|
end
|
@@ -37,6 +43,16 @@ shared_context 'with a work type' do
|
|
37
43
|
end
|
38
44
|
end
|
39
45
|
|
46
|
+
class Institution
|
47
|
+
def self.name
|
48
|
+
'my_institution'
|
49
|
+
end
|
50
|
+
|
51
|
+
def self.name_full
|
52
|
+
'my full institution name'
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
40
56
|
class UploadedFile < ActiveFedora::Base
|
41
57
|
def self.create(*)
|
42
58
|
h = Hyrax::UploadedFile.new
|
@@ -47,14 +63,20 @@ shared_context 'with a work type' do
|
|
47
63
|
|
48
64
|
module Actors
|
49
65
|
class Environment
|
50
|
-
|
66
|
+
attr_reader :new_object, :attributes
|
67
|
+
def initialize(new_object, _ability, attributes)
|
68
|
+
@new_object = new_object
|
69
|
+
@attributes = attributes
|
70
|
+
end
|
51
71
|
end
|
52
72
|
end
|
53
73
|
|
54
74
|
class Actor
|
55
|
-
def create(
|
56
|
-
|
57
|
-
|
75
|
+
def create(actor_env)
|
76
|
+
attrs = actor_env.attributes
|
77
|
+
attrs.delete(:uploaded_files)
|
78
|
+
actor_env.new_object.attributes = attrs
|
79
|
+
actor_env.new_object.save
|
58
80
|
end
|
59
81
|
end
|
60
82
|
|
@@ -68,6 +90,8 @@ shared_context 'with a work type' do
|
|
68
90
|
|
69
91
|
after do
|
70
92
|
Object.send(:remove_const, :Hyrax) if defined?(Hyrax)
|
71
|
-
Object.send(:remove_const, :Work)
|
93
|
+
Object.send(:remove_const, :Work) if defined?(Work)
|
94
|
+
Object.send(:remove_const, :User) if defined?(User)
|
95
|
+
Object.send(:remove_const, :Ability) if defined?(Ability)
|
72
96
|
end
|
73
97
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: darlingtonia
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Data Curation Experts
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-02-
|
11
|
+
date: 2019-02-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: active-fedora
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '0
|
47
|
+
version: '1.0'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '0
|
54
|
+
version: '1.0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: hyrax-spec
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -136,6 +136,20 @@ dependencies:
|
|
136
136
|
- - ">="
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '0'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: rake
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - ">="
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '0'
|
146
|
+
type: :development
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - ">="
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '0'
|
139
153
|
description:
|
140
154
|
email:
|
141
155
|
- administrator@curationexperts.com
|
@@ -160,6 +174,7 @@ files:
|
|
160
174
|
- lib/darlingtonia/hyrax_record_importer.rb
|
161
175
|
- lib/darlingtonia/importer.rb
|
162
176
|
- lib/darlingtonia/input_record.rb
|
177
|
+
- lib/darlingtonia/log_stream.rb
|
163
178
|
- lib/darlingtonia/metadata_mapper.rb
|
164
179
|
- lib/darlingtonia/parser.rb
|
165
180
|
- lib/darlingtonia/parsers/csv_parser.rb
|