bulkrax 1.0.2 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/app/controllers/bulkrax/exporters_controller.rb +12 -4
- data/app/controllers/bulkrax/importers_controller.rb +22 -17
- data/app/factories/bulkrax/object_factory.rb +36 -59
- data/app/jobs/bulkrax/create_relationships_job.rb +187 -0
- data/app/jobs/bulkrax/delete_work_job.rb +6 -2
- data/app/jobs/bulkrax/export_work_job.rb +3 -1
- data/app/jobs/bulkrax/exporter_job.rb +1 -0
- data/app/jobs/bulkrax/{import_work_collection_job.rb → import_collection_job.rb} +2 -2
- data/app/jobs/bulkrax/importer_job.rb +16 -1
- data/app/matchers/bulkrax/application_matcher.rb +5 -5
- data/app/models/bulkrax/csv_collection_entry.rb +8 -6
- data/app/models/bulkrax/csv_entry.rb +124 -66
- data/app/models/bulkrax/entry.rb +19 -8
- data/app/models/bulkrax/exporter.rb +12 -5
- data/app/models/bulkrax/importer.rb +12 -5
- data/app/models/bulkrax/oai_entry.rb +5 -1
- data/app/models/bulkrax/rdf_entry.rb +16 -7
- data/app/models/bulkrax/xml_entry.rb +4 -0
- data/app/models/concerns/bulkrax/export_behavior.rb +2 -2
- data/app/models/concerns/bulkrax/has_matchers.rb +44 -13
- data/app/models/concerns/bulkrax/import_behavior.rb +35 -5
- data/app/models/concerns/bulkrax/importer_exporter_behavior.rb +19 -0
- data/app/models/concerns/bulkrax/status_info.rb +4 -4
- data/app/parsers/bulkrax/application_parser.rb +59 -84
- data/app/parsers/bulkrax/bagit_parser.rb +12 -3
- data/app/parsers/bulkrax/csv_parser.rb +117 -63
- data/app/parsers/bulkrax/oai_dc_parser.rb +5 -2
- data/app/parsers/bulkrax/xml_parser.rb +5 -0
- data/app/views/bulkrax/exporters/_form.html.erb +1 -1
- data/app/views/bulkrax/exporters/show.html.erb +2 -1
- data/app/views/bulkrax/importers/index.html.erb +15 -17
- data/app/views/bulkrax/importers/show.html.erb +6 -2
- data/config/locales/bulkrax.en.yml +1 -0
- data/db/migrate/20190731114016_change_importer_and_exporter_to_polymorphic.rb +5 -1
- data/db/migrate/20211004170708_change_bulkrax_statuses_error_message_column_type_to_text.rb +5 -0
- data/db/migrate/20211203195233_rename_children_counters_to_relationships.rb +6 -0
- data/lib/bulkrax/engine.rb +1 -1
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +9 -17
- data/lib/generators/bulkrax/templates/bin/importer +17 -11
- data/lib/generators/bulkrax/templates/config/bulkrax_api.yml +3 -1
- data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb +7 -12
- metadata +6 -4
- data/app/jobs/bulkrax/child_relationships_job.rb +0 -128
@@ -14,59 +14,66 @@ module Bulkrax
|
|
14
14
|
def self.read_data(path)
|
15
15
|
raise StandardError, 'CSV path empty' if path.blank?
|
16
16
|
CSV.read(path,
|
17
|
-
|
18
|
-
|
19
|
-
|
17
|
+
headers: true,
|
18
|
+
header_converters: :symbol,
|
19
|
+
encoding: 'utf-8')
|
20
20
|
end
|
21
21
|
|
22
22
|
def self.data_for_entry(data, _source_id)
|
23
|
+
ActiveSupport::Deprecation.warn(
|
24
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
25
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
26
|
+
)
|
23
27
|
# If a multi-line CSV data is passed, grab the first row
|
24
28
|
data = data.first if data.is_a?(CSV::Table)
|
25
29
|
# model has to be separated so that it doesn't get mistranslated by to_h
|
26
30
|
raw_data = data.to_h
|
27
|
-
raw_data[:model] = data[:model]
|
31
|
+
raw_data[:model] = data[:model] if data[:model].present?
|
28
32
|
# If the collection field mapping is not 'collection', add 'collection' - the parser needs it
|
29
33
|
raw_data[:collection] = raw_data[collection_field.to_sym] if raw_data.keys.include?(collection_field.to_sym) && collection_field != 'collection'
|
30
|
-
# If the children field mapping is not 'children', add 'children' - the parser needs it
|
31
|
-
raw_data[:children] = raw_data[collection_field.to_sym] if raw_data.keys.include?(children_field.to_sym) && children_field != 'children'
|
32
34
|
return raw_data
|
33
35
|
end
|
34
36
|
|
35
|
-
def
|
36
|
-
|
37
|
-
|
37
|
+
def build_metadata
|
38
|
+
raise StandardError, 'Record not found' if record.nil?
|
39
|
+
raise StandardError, "Missing required elements, missing element(s) are: #{importerexporter.parser.missing_elements(keys_without_numbers(record.keys)).join(', ')}" unless importerexporter.parser.required_elements?(keys_without_numbers(record.keys))
|
38
40
|
|
39
|
-
|
40
|
-
|
41
|
-
|
41
|
+
self.parsed_metadata = {}
|
42
|
+
add_identifier
|
43
|
+
add_metadata_for_model
|
44
|
+
add_visibility
|
45
|
+
add_ingested_metadata
|
46
|
+
add_rights_statement
|
47
|
+
add_collections
|
48
|
+
add_local
|
42
49
|
|
43
|
-
|
44
|
-
keys.map { |key| key_without_numbers(key) }
|
50
|
+
self.parsed_metadata
|
45
51
|
end
|
46
52
|
|
47
|
-
def
|
48
|
-
|
53
|
+
def add_identifier
|
54
|
+
self.parsed_metadata[work_identifier] = [record[source_identifier]]
|
49
55
|
end
|
50
56
|
|
51
|
-
def
|
52
|
-
|
53
|
-
|
57
|
+
def add_metadata_for_model
|
58
|
+
if factory_class == Collection
|
59
|
+
add_collection_type_gid
|
60
|
+
else
|
61
|
+
add_file unless importerexporter.metadata_only?
|
62
|
+
add_admin_set_id
|
63
|
+
end
|
64
|
+
end
|
54
65
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
66
|
+
def add_ingested_metadata
|
67
|
+
ActiveSupport::Deprecation.warn(
|
68
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
69
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
70
|
+
)
|
71
|
+
record.sort.each do |key, value|
|
72
|
+
next if self.parser.collection_field_mapping.to_s == key_without_numbers(key)
|
59
73
|
|
60
74
|
index = key[/\d+/].to_i - 1 if key[/\d+/].to_i != 0
|
61
75
|
add_metadata(key_without_numbers(key), value, index)
|
62
76
|
end
|
63
|
-
add_file
|
64
|
-
add_visibility
|
65
|
-
add_rights_statement
|
66
|
-
add_admin_set_id
|
67
|
-
add_collections
|
68
|
-
add_local
|
69
|
-
self.parsed_metadata
|
70
77
|
end
|
71
78
|
|
72
79
|
def add_file
|
@@ -86,10 +93,20 @@ module Bulkrax
|
|
86
93
|
self.parsed_metadata[source_identifier] = hyrax_record.send(work_identifier)
|
87
94
|
self.parsed_metadata['model'] = hyrax_record.has_model.first
|
88
95
|
build_mapping_metadata
|
89
|
-
|
90
|
-
|
91
|
-
|
96
|
+
|
97
|
+
# TODO: fix the "send" parameter in the conditional below
|
98
|
+
# currently it returns: "NoMethodError - undefined method 'bulkrax_identifier' for #<Collection:0x00007fbe6a3b4248>"
|
99
|
+
if mapping['collection']&.[]('join')
|
100
|
+
self.parsed_metadata['collection'] = hyrax_record.member_of_collection_ids.join('; ')
|
101
|
+
# self.parsed_metadata['collection'] = hyrax_record.member_of_collections.map { |c| c.send(work_identifier)&.first }.compact.uniq.join(';')
|
102
|
+
else
|
103
|
+
hyrax_record.member_of_collections.each_with_index do |collection, i|
|
104
|
+
self.parsed_metadata["collection_#{i + 1}"] = collection.id
|
105
|
+
# self.parsed_metadata["collection_#{i + 1}"] = collection.send(work_identifier)&.first
|
106
|
+
end
|
92
107
|
end
|
108
|
+
|
109
|
+
build_files unless hyrax_record.is_a?(Collection)
|
93
110
|
self.parsed_metadata
|
94
111
|
end
|
95
112
|
|
@@ -97,24 +114,51 @@ module Bulkrax
|
|
97
114
|
mapping.each do |key, value|
|
98
115
|
next if Bulkrax.reserved_properties.include?(key) && !field_supported?(key)
|
99
116
|
next if key == "model"
|
117
|
+
next if value['excluded']
|
100
118
|
|
101
119
|
object_key = key if value.key?('object')
|
102
120
|
next unless hyrax_record.respond_to?(key.to_s) || object_key.present?
|
103
121
|
|
104
|
-
data = object_key.present? ? hyrax_record.send(value['object']) : hyrax_record.send(key.to_s)
|
105
122
|
if object_key.present?
|
106
|
-
|
107
|
-
|
123
|
+
build_object(value)
|
124
|
+
else
|
125
|
+
build_value(key, value)
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def build_object(value)
|
131
|
+
data = hyrax_record.send(value['object'])
|
132
|
+
return if data.empty?
|
133
|
+
|
134
|
+
data = data.to_a if data.is_a?(ActiveTriples::Relation)
|
135
|
+
object_metadata(Array.wrap(data))
|
136
|
+
end
|
108
137
|
|
109
|
-
|
110
|
-
|
111
|
-
|
138
|
+
def build_value(key, value)
|
139
|
+
data = hyrax_record.send(key.to_s)
|
140
|
+
if data.is_a?(ActiveTriples::Relation)
|
141
|
+
if value['join']
|
142
|
+
self.parsed_metadata[key_for_export(key)] = data.map { |d| prepare_export_data(d) }.join('; ').to_s
|
112
143
|
else
|
113
|
-
|
144
|
+
data.each_with_index do |d, i|
|
145
|
+
self.parsed_metadata["#{key_for_export(key)}_#{i + 1}"] = prepare_export_data(d)
|
146
|
+
end
|
114
147
|
end
|
148
|
+
else
|
149
|
+
self.parsed_metadata[key_for_export(key)] = prepare_export_data(data)
|
115
150
|
end
|
116
151
|
end
|
117
152
|
|
153
|
+
# On export the key becomes the from and the from becomes the destination. It is the opposite of the import because we are moving data the opposite direction
|
154
|
+
# metadata that does not have a specific Bulkrax entry is mapped to the key name, as matching keys coming in are mapped by the csv parser automatically
|
155
|
+
def key_for_export(key)
|
156
|
+
clean_key = key_without_numbers(key)
|
157
|
+
unnumbered_key = mapping[clean_key] ? mapping[clean_key]['from'].first : clean_key
|
158
|
+
# Bring the number back if there is one
|
159
|
+
"#{unnumbered_key}#{key.sub(clean_key, '')}"
|
160
|
+
end
|
161
|
+
|
118
162
|
def prepare_export_data(datum)
|
119
163
|
if datum.is_a?(ActiveTriples::Resource)
|
120
164
|
datum.to_uri.to_s
|
@@ -123,30 +167,34 @@ module Bulkrax
|
|
123
167
|
end
|
124
168
|
end
|
125
169
|
|
126
|
-
def object_metadata(data
|
127
|
-
data =
|
170
|
+
def object_metadata(data)
|
171
|
+
data = data.map { |d| eval(d) }.flatten # rubocop:disable Security/Eval
|
128
172
|
|
129
173
|
data.each_with_index do |obj, index|
|
130
|
-
next
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
obj
|
135
|
-
|
174
|
+
next if obj.nil?
|
175
|
+
# allow the object_key to be valid whether it's a string or symbol
|
176
|
+
obj = obj.with_indifferent_access
|
177
|
+
|
178
|
+
obj.each_key do |key|
|
179
|
+
if obj[key].is_a?(Array)
|
180
|
+
obj[key].each_with_index do |_nested_item, nested_index|
|
181
|
+
self.parsed_metadata["#{key_for_export(key)}_#{index + 1}_#{nested_index + 1}"] = prepare_export_data(obj[key][nested_index])
|
182
|
+
end
|
183
|
+
else
|
184
|
+
self.parsed_metadata["#{key_for_export(key)}_#{index + 1}"] = prepare_export_data(obj[key])
|
185
|
+
end
|
136
186
|
end
|
137
187
|
end
|
138
188
|
end
|
139
189
|
|
140
|
-
def
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
return gsub_data.map { |d| JSON.parse(d) }
|
190
|
+
def build_files
|
191
|
+
if mapping['file']&.[]('join')
|
192
|
+
self.parsed_metadata['file'] = hyrax_record.file_sets.map { |fs| filename(fs).to_s if filename(fs).present? }.compact.join('; ')
|
193
|
+
else
|
194
|
+
hyrax_record.file_sets.each_with_index do |fs, i|
|
195
|
+
self.parsed_metadata["file_#{i + 1}"] = filename(fs).to_s if filename(fs).present?
|
196
|
+
end
|
197
|
+
end
|
150
198
|
end
|
151
199
|
|
152
200
|
# In order for the existing exported hyrax_record, to be updated by a re-import
|
@@ -167,18 +215,28 @@ module Bulkrax
|
|
167
215
|
Bulkrax::CsvMatcher
|
168
216
|
end
|
169
217
|
|
218
|
+
def possible_collection_ids
|
219
|
+
ActiveSupport::Deprecation.warn(
|
220
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
221
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
222
|
+
)
|
223
|
+
@possible_collection_ids ||= record.inject([]) do |memo, (key, value)|
|
224
|
+
memo += value.split(/\s*[:;|]\s*/) if self.class.collection_field.to_s == key_without_numbers(key) && value.present?
|
225
|
+
memo
|
226
|
+
end || []
|
227
|
+
end
|
228
|
+
|
170
229
|
def collections_created?
|
171
|
-
|
172
|
-
record[self.class.collection_field].split(/\s*[:;|]\s*/).length == self.collection_ids.length
|
230
|
+
possible_collection_ids.length == self.collection_ids.length
|
173
231
|
end
|
174
232
|
|
175
|
-
def
|
233
|
+
def find_collection_ids
|
176
234
|
return self.collection_ids if collections_created?
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
self.collection_ids << c.id unless
|
235
|
+
if possible_collection_ids.present?
|
236
|
+
possible_collection_ids.each do |collection_id|
|
237
|
+
c = find_collection(collection_id)
|
238
|
+
skip = c.blank? || self.collection_ids.include?(c.id)
|
239
|
+
self.collection_ids << c.id unless skip
|
182
240
|
end
|
183
241
|
end
|
184
242
|
self.collection_ids
|
data/app/models/bulkrax/entry.rb
CHANGED
@@ -23,12 +23,22 @@ module Bulkrax
|
|
23
23
|
|
24
24
|
attr_accessor :all_attrs
|
25
25
|
|
26
|
-
delegate :parser,
|
26
|
+
delegate :parser,
|
27
|
+
:mapping,
|
28
|
+
:replace_files,
|
29
|
+
:update_files,
|
30
|
+
:keys_without_numbers,
|
31
|
+
:key_without_numbers,
|
32
|
+
to: :importerexporter
|
27
33
|
|
28
34
|
delegate :client,
|
29
|
-
|
30
|
-
|
31
|
-
|
35
|
+
:collection_name,
|
36
|
+
:user,
|
37
|
+
:related_parents_raw_mapping,
|
38
|
+
:related_parents_parsed_mapping,
|
39
|
+
:related_children_raw_mapping,
|
40
|
+
:related_children_parsed_mapping,
|
41
|
+
to: :parser
|
32
42
|
|
33
43
|
# Retrieve fields from the file
|
34
44
|
# @param data - the source data
|
@@ -61,13 +71,13 @@ module Bulkrax
|
|
61
71
|
end
|
62
72
|
|
63
73
|
def self.collection_field
|
74
|
+
ActiveSupport::Deprecation.warn(
|
75
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
76
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
77
|
+
)
|
64
78
|
Bulkrax.collection_field_mapping[self.to_s]
|
65
79
|
end
|
66
80
|
|
67
|
-
def self.children_field
|
68
|
-
Bulkrax.parent_child_field_mapping[self.to_s]
|
69
|
-
end
|
70
|
-
|
71
81
|
def build
|
72
82
|
return if type.nil?
|
73
83
|
self.save if self.new_record? # must be saved for statuses
|
@@ -96,6 +106,7 @@ module Bulkrax
|
|
96
106
|
end
|
97
107
|
|
98
108
|
def find_collection(collection_identifier)
|
109
|
+
return unless Collection.properties.keys.include?(work_identifier)
|
99
110
|
Collection.where(
|
100
111
|
work_identifier => collection_identifier
|
101
112
|
).detect { |m| m.send(work_identifier).include?(collection_identifier) }
|
@@ -14,7 +14,7 @@ module Bulkrax
|
|
14
14
|
validates :name, presence: true
|
15
15
|
validates :parser_klass, presence: true
|
16
16
|
|
17
|
-
delegate :write, :create_from_collection, :create_from_importer, :create_from_worktype, to: :parser
|
17
|
+
delegate :write, :create_from_collection, :create_from_importer, :create_from_worktype, :create_from_all, to: :parser
|
18
18
|
|
19
19
|
def export
|
20
20
|
current_run && setup_export_path
|
@@ -25,6 +25,8 @@ module Bulkrax
|
|
25
25
|
create_from_importer
|
26
26
|
when 'worktype'
|
27
27
|
create_from_worktype
|
28
|
+
when 'all'
|
29
|
+
create_from_all
|
28
30
|
end
|
29
31
|
rescue StandardError => e
|
30
32
|
status_info(e)
|
@@ -77,7 +79,8 @@ module Bulkrax
|
|
77
79
|
[
|
78
80
|
[I18n.t('bulkrax.exporter.labels.importer'), 'importer'],
|
79
81
|
[I18n.t('bulkrax.exporter.labels.collection'), 'collection'],
|
80
|
-
[I18n.t('bulkrax.exporter.labels.worktype'), 'worktype']
|
82
|
+
[I18n.t('bulkrax.exporter.labels.worktype'), 'worktype'],
|
83
|
+
[I18n.t('bulkrax.exporter.labels.all'), 'all']
|
81
84
|
]
|
82
85
|
end
|
83
86
|
|
@@ -105,18 +108,22 @@ module Bulkrax
|
|
105
108
|
end
|
106
109
|
|
107
110
|
def exporter_export_path
|
108
|
-
@exporter_export_path ||= File.join(
|
111
|
+
@exporter_export_path ||= File.join(parser.base_path('export'), self.id.to_s, self.exporter_runs.last.id.to_s)
|
109
112
|
end
|
110
113
|
|
111
114
|
def exporter_export_zip_path
|
112
|
-
@exporter_export_zip_path ||= File.join(
|
115
|
+
@exporter_export_zip_path ||= File.join(parser.base_path('export'), "export_#{self.id}_#{self.exporter_runs.last.id}.zip")
|
113
116
|
rescue
|
114
|
-
@exporter_export_zip_path ||= File.join(
|
117
|
+
@exporter_export_zip_path ||= File.join(parser.base_path('export'), "export_#{self.id}_0.zip")
|
115
118
|
end
|
116
119
|
|
117
120
|
def export_properties
|
118
121
|
properties = Hyrax.config.registered_curation_concern_types.map { |work| work.constantize.properties.keys }.flatten.uniq.sort
|
119
122
|
properties.reject { |prop| Bulkrax.reserved_properties.include?(prop) }
|
120
123
|
end
|
124
|
+
|
125
|
+
def metadata_only?
|
126
|
+
export_type == 'metadata'
|
127
|
+
end
|
121
128
|
end
|
122
129
|
end
|
@@ -18,8 +18,7 @@ module Bulkrax
|
|
18
18
|
validates :admin_set_id, presence: true
|
19
19
|
validates :parser_klass, presence: true
|
20
20
|
|
21
|
-
delegate :valid_import?, :
|
22
|
-
:write_errored_entries_file, :visibility, to: :parser
|
21
|
+
delegate :valid_import?, :write_errored_entries_file, :visibility, to: :parser
|
23
22
|
|
24
23
|
attr_accessor :only_updates, :file_style, :file
|
25
24
|
attr_writer :current_run
|
@@ -97,7 +96,11 @@ module Bulkrax
|
|
97
96
|
end
|
98
97
|
|
99
98
|
def current_run
|
100
|
-
@current_run ||=
|
99
|
+
@current_run ||= if file? && zip?
|
100
|
+
self.importer_runs.create!
|
101
|
+
else
|
102
|
+
self.importer_runs.create!(total_work_entries: self.limit || parser.works_total, total_collection_entries: parser.collections_total)
|
103
|
+
end
|
101
104
|
end
|
102
105
|
|
103
106
|
def last_run
|
@@ -149,11 +152,11 @@ module Bulkrax
|
|
149
152
|
|
150
153
|
# If the import data is zipped, unzip it to this path
|
151
154
|
def importer_unzip_path
|
152
|
-
@importer_unzip_path ||= File.join(
|
155
|
+
@importer_unzip_path ||= File.join(parser.base_path, "import_#{path_string}")
|
153
156
|
end
|
154
157
|
|
155
158
|
def errored_entries_csv_path
|
156
|
-
@errored_entries_csv_path ||= File.join(
|
159
|
+
@errored_entries_csv_path ||= File.join(parser.base_path, "import_#{path_string}_errored_entries.csv")
|
157
160
|
end
|
158
161
|
|
159
162
|
def path_string
|
@@ -161,5 +164,9 @@ module Bulkrax
|
|
161
164
|
rescue
|
162
165
|
"#{self.id}_#{self.created_at.strftime('%Y%m%d%H%M%S')}"
|
163
166
|
end
|
167
|
+
|
168
|
+
def metadata_only?
|
169
|
+
parser.parser_fields['metadata_only'] == true
|
170
|
+
end
|
164
171
|
end
|
165
172
|
end
|
@@ -26,6 +26,10 @@ module Bulkrax
|
|
26
26
|
end
|
27
27
|
|
28
28
|
def build_metadata
|
29
|
+
ActiveSupport::Deprecation.warn(
|
30
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
31
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
32
|
+
)
|
29
33
|
self.parsed_metadata = {}
|
30
34
|
self.parsed_metadata[work_identifier] = [record.header.identifier]
|
31
35
|
|
@@ -56,7 +60,7 @@ module Bulkrax
|
|
56
60
|
# Retrieve list of collections for the entry; add to collection_ids
|
57
61
|
# If OAI-PMH doesn't return setSpec in the headers for GetRecord, use parser.collection_name
|
58
62
|
# in this case, if 'All' is selected, records will not be added to a collection.
|
59
|
-
def
|
63
|
+
def find_collection_ids
|
60
64
|
return self.collection_ids if collections_created?
|
61
65
|
if sets.blank? || parser.collection_name != 'all'
|
62
66
|
# c = Collection.where(Bulkrax.system_identifier_field => importerexporter.unique_collection_identifier(parser.collection_name)).first
|
@@ -14,6 +14,10 @@ module Bulkrax
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def self.data_for_entry(data, source_id)
|
17
|
+
ActiveSupport::Deprecation.warn(
|
18
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
19
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
20
|
+
)
|
17
21
|
reader = data
|
18
22
|
format = reader.class.format.to_sym
|
19
23
|
collections = []
|
@@ -22,7 +26,7 @@ module Bulkrax
|
|
22
26
|
data = RDF::Writer.for(format).buffer do |writer|
|
23
27
|
reader.each_statement do |statement|
|
24
28
|
collections << statement.object.to_s if collection_field.present? && collection_field == statement.predicate.to_s
|
25
|
-
children << statement.object.to_s if
|
29
|
+
children << statement.object.to_s if related_children_parsed_mapping.present? && related_children_parsed_mapping == statement.predicate.to_s
|
26
30
|
delete = statement.object.to_s if /deleted/.match?(statement.predicate.to_s)
|
27
31
|
writer << statement
|
28
32
|
end
|
@@ -37,12 +41,13 @@ module Bulkrax
|
|
37
41
|
}
|
38
42
|
end
|
39
43
|
|
40
|
-
def self.
|
41
|
-
|
42
|
-
|
44
|
+
def self.related_children_parsed_mapping
|
45
|
+
return @related_children_parsed_mapping if @related_children_parsed_mapping.present?
|
46
|
+
|
47
|
+
rdf_related_children_field_mapping = Bulkrax.field_mappings['Bulkrax::RdfParser']&.select { |_, h| h.key?('related_children_field_mapping') }
|
48
|
+
return if rdf_related_children_field_mapping.blank?
|
43
49
|
|
44
|
-
|
45
|
-
Bulkrax.parent_child_field_mapping[self.to_s]
|
50
|
+
@related_children_parsed_mapping = rdf_related_children_field_mapping&.keys&.first
|
46
51
|
end
|
47
52
|
|
48
53
|
def record
|
@@ -50,6 +55,10 @@ module Bulkrax
|
|
50
55
|
end
|
51
56
|
|
52
57
|
def build_metadata
|
58
|
+
ActiveSupport::Deprecation.warn(
|
59
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
60
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
61
|
+
)
|
53
62
|
raise StandardError, 'Record not found' if record.nil?
|
54
63
|
raise StandardError, "Missing source identifier (#{source_identifier})" if self.raw_metadata[source_identifier].blank?
|
55
64
|
|
@@ -76,7 +85,7 @@ module Bulkrax
|
|
76
85
|
self.raw_metadata['collection'].length == self.collection_ids.length
|
77
86
|
end
|
78
87
|
|
79
|
-
def
|
88
|
+
def find_collection_ids
|
80
89
|
return self.collection_ids if collections_created?
|
81
90
|
if self.raw_metadata['collection'].present?
|
82
91
|
self.raw_metadata['collection'].each do |collection|
|
@@ -39,6 +39,10 @@ module Bulkrax
|
|
39
39
|
end
|
40
40
|
|
41
41
|
def build_metadata
|
42
|
+
ActiveSupport::Deprecation.warn(
|
43
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
44
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
45
|
+
)
|
42
46
|
raise StandardError, 'Record not found' if record.nil?
|
43
47
|
raise StandardError, "Missing source identifier (#{source_identifier})" if self.raw_metadata[source_identifier].blank?
|
44
48
|
self.parsed_metadata = {}
|
@@ -40,13 +40,13 @@ module Bulkrax
|
|
40
40
|
end
|
41
41
|
end
|
42
42
|
|
43
|
-
#
|
43
|
+
# Prepend the file_set id to ensure a unique filename
|
44
44
|
def filename(file_set)
|
45
45
|
return if file_set.original_file.blank?
|
46
46
|
fn = file_set.original_file.file_name.first
|
47
47
|
mime = Mime::Type.lookup(file_set.original_file.mime_type)
|
48
48
|
ext_mime = MIME::Types.of(file_set.original_file.file_name).first
|
49
|
-
if fn.include?(file_set.id)
|
49
|
+
if fn.include?(file_set.id) || importerexporter.metadata_only?
|
50
50
|
return fn if mime.to_s == ext_mime.to_s
|
51
51
|
return "#{fn}.#{mime.to_sym}"
|
52
52
|
else
|
@@ -43,7 +43,6 @@ module Bulkrax
|
|
43
43
|
|
44
44
|
value = if matcher
|
45
45
|
result = matcher.result(self, node_content)
|
46
|
-
next unless result
|
47
46
|
matched_metadata(multiple, name, result, object_multiple)
|
48
47
|
elsif multiple
|
49
48
|
Rails.logger.info("Bulkrax Column automatically matched #{node_name}, #{node_content}")
|
@@ -53,12 +52,20 @@ module Bulkrax
|
|
53
52
|
single_metadata(node_content)
|
54
53
|
end
|
55
54
|
|
56
|
-
|
55
|
+
object_name.present? ? set_parsed_object_data(object_multiple, object_name, name, index, value) : set_parsed_data(name, value)
|
57
56
|
end
|
58
57
|
end
|
59
58
|
|
60
|
-
def set_parsed_data(
|
59
|
+
def set_parsed_data(name, value)
|
60
|
+
return parsed_metadata[name] = value unless multiple?(name)
|
61
|
+
|
62
|
+
parsed_metadata[name] ||= []
|
63
|
+
parsed_metadata[name] += Array.wrap(value).flatten
|
64
|
+
end
|
65
|
+
|
66
|
+
def set_parsed_object_data(object_multiple, object_name, name, index, value)
|
61
67
|
if object_multiple
|
68
|
+
index ||= 0
|
62
69
|
parsed_metadata[object_name][index] ||= {}
|
63
70
|
parsed_metadata[object_name][index][name] ||= []
|
64
71
|
if value.is_a?(Array)
|
@@ -66,20 +73,13 @@ module Bulkrax
|
|
66
73
|
else
|
67
74
|
parsed_metadata[object_name][index][name] = value
|
68
75
|
end
|
69
|
-
|
76
|
+
else
|
70
77
|
parsed_metadata[object_name][name] ||= []
|
71
78
|
if value.is_a?(Array)
|
72
79
|
parsed_metadata[object_name][name] += value
|
73
80
|
else
|
74
81
|
parsed_metadata[object_name][name] = value
|
75
82
|
end
|
76
|
-
else
|
77
|
-
parsed_metadata[name] ||= []
|
78
|
-
if value.is_a?(Array)
|
79
|
-
parsed_metadata[name] += value
|
80
|
-
else
|
81
|
-
parsed_metadata[name] = value
|
82
|
-
end
|
83
83
|
end
|
84
84
|
end
|
85
85
|
|
@@ -124,12 +124,43 @@ module Bulkrax
|
|
124
124
|
field = field.gsub('_attributes', '')
|
125
125
|
|
126
126
|
return false if excluded?(field)
|
127
|
-
return true if
|
127
|
+
return true if supported_bulkrax_fields.include?(field)
|
128
128
|
return factory_class.method_defined?(field) && factory_class.properties[field].present?
|
129
129
|
end
|
130
130
|
|
131
|
+
def supported_bulkrax_fields
|
132
|
+
ActiveSupport::Deprecation.warn(
|
133
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
134
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
135
|
+
)
|
136
|
+
@supported_bulkrax_fields ||=
|
137
|
+
%W[
|
138
|
+
id
|
139
|
+
file
|
140
|
+
remote_files
|
141
|
+
model
|
142
|
+
delete
|
143
|
+
#{parser.collection_field_mapping}
|
144
|
+
#{related_parents_parsed_mapping}
|
145
|
+
#{related_children_parsed_mapping}
|
146
|
+
]
|
147
|
+
end
|
148
|
+
|
131
149
|
def multiple?(field)
|
132
|
-
|
150
|
+
ActiveSupport::Deprecation.warn(
|
151
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
152
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
153
|
+
)
|
154
|
+
@multiple_bulkrax_fields ||=
|
155
|
+
%W[
|
156
|
+
file
|
157
|
+
remote_files
|
158
|
+
#{parser.collection_field_mapping}
|
159
|
+
#{related_parents_parsed_mapping}
|
160
|
+
#{related_children_parsed_mapping}
|
161
|
+
]
|
162
|
+
|
163
|
+
return true if @multiple_bulkrax_fields.include?(field)
|
133
164
|
return false if field == 'model'
|
134
165
|
|
135
166
|
field_supported?(field) && factory_class&.properties&.[](field)&.[]('multiple')
|