bulkrax 1.0.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/app/controllers/bulkrax/exporters_controller.rb +12 -4
- data/app/controllers/bulkrax/importers_controller.rb +22 -17
- data/app/factories/bulkrax/object_factory.rb +36 -59
- data/app/jobs/bulkrax/create_relationships_job.rb +187 -0
- data/app/jobs/bulkrax/delete_work_job.rb +6 -2
- data/app/jobs/bulkrax/export_work_job.rb +3 -1
- data/app/jobs/bulkrax/exporter_job.rb +1 -0
- data/app/jobs/bulkrax/{import_work_collection_job.rb → import_collection_job.rb} +2 -2
- data/app/jobs/bulkrax/importer_job.rb +16 -1
- data/app/matchers/bulkrax/application_matcher.rb +5 -5
- data/app/models/bulkrax/csv_collection_entry.rb +8 -6
- data/app/models/bulkrax/csv_entry.rb +124 -66
- data/app/models/bulkrax/entry.rb +19 -8
- data/app/models/bulkrax/exporter.rb +12 -5
- data/app/models/bulkrax/importer.rb +12 -5
- data/app/models/bulkrax/oai_entry.rb +5 -1
- data/app/models/bulkrax/rdf_entry.rb +16 -7
- data/app/models/bulkrax/xml_entry.rb +4 -0
- data/app/models/concerns/bulkrax/export_behavior.rb +2 -2
- data/app/models/concerns/bulkrax/has_matchers.rb +44 -13
- data/app/models/concerns/bulkrax/import_behavior.rb +35 -5
- data/app/models/concerns/bulkrax/importer_exporter_behavior.rb +19 -0
- data/app/models/concerns/bulkrax/status_info.rb +4 -4
- data/app/parsers/bulkrax/application_parser.rb +59 -84
- data/app/parsers/bulkrax/bagit_parser.rb +12 -3
- data/app/parsers/bulkrax/csv_parser.rb +117 -63
- data/app/parsers/bulkrax/oai_dc_parser.rb +5 -2
- data/app/parsers/bulkrax/xml_parser.rb +5 -0
- data/app/views/bulkrax/exporters/_form.html.erb +1 -1
- data/app/views/bulkrax/exporters/show.html.erb +2 -1
- data/app/views/bulkrax/importers/index.html.erb +15 -17
- data/app/views/bulkrax/importers/show.html.erb +6 -2
- data/config/locales/bulkrax.en.yml +1 -0
- data/db/migrate/20190731114016_change_importer_and_exporter_to_polymorphic.rb +5 -1
- data/db/migrate/20211004170708_change_bulkrax_statuses_error_message_column_type_to_text.rb +5 -0
- data/db/migrate/20211203195233_rename_children_counters_to_relationships.rb +6 -0
- data/lib/bulkrax/engine.rb +1 -1
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +9 -17
- data/lib/generators/bulkrax/templates/bin/importer +17 -11
- data/lib/generators/bulkrax/templates/config/bulkrax_api.yml +3 -1
- data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb +7 -12
- metadata +6 -4
- data/app/jobs/bulkrax/child_relationships_job.rb +0 -128
@@ -14,59 +14,66 @@ module Bulkrax
|
|
14
14
|
def self.read_data(path)
|
15
15
|
raise StandardError, 'CSV path empty' if path.blank?
|
16
16
|
CSV.read(path,
|
17
|
-
|
18
|
-
|
19
|
-
|
17
|
+
headers: true,
|
18
|
+
header_converters: :symbol,
|
19
|
+
encoding: 'utf-8')
|
20
20
|
end
|
21
21
|
|
22
22
|
def self.data_for_entry(data, _source_id)
|
23
|
+
ActiveSupport::Deprecation.warn(
|
24
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
25
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
26
|
+
)
|
23
27
|
# If a multi-line CSV data is passed, grab the first row
|
24
28
|
data = data.first if data.is_a?(CSV::Table)
|
25
29
|
# model has to be separated so that it doesn't get mistranslated by to_h
|
26
30
|
raw_data = data.to_h
|
27
|
-
raw_data[:model] = data[:model]
|
31
|
+
raw_data[:model] = data[:model] if data[:model].present?
|
28
32
|
# If the collection field mapping is not 'collection', add 'collection' - the parser needs it
|
29
33
|
raw_data[:collection] = raw_data[collection_field.to_sym] if raw_data.keys.include?(collection_field.to_sym) && collection_field != 'collection'
|
30
|
-
# If the children field mapping is not 'children', add 'children' - the parser needs it
|
31
|
-
raw_data[:children] = raw_data[collection_field.to_sym] if raw_data.keys.include?(children_field.to_sym) && children_field != 'children'
|
32
34
|
return raw_data
|
33
35
|
end
|
34
36
|
|
35
|
-
def
|
36
|
-
|
37
|
-
|
37
|
+
def build_metadata
|
38
|
+
raise StandardError, 'Record not found' if record.nil?
|
39
|
+
raise StandardError, "Missing required elements, missing element(s) are: #{importerexporter.parser.missing_elements(keys_without_numbers(record.keys)).join(', ')}" unless importerexporter.parser.required_elements?(keys_without_numbers(record.keys))
|
38
40
|
|
39
|
-
|
40
|
-
|
41
|
-
|
41
|
+
self.parsed_metadata = {}
|
42
|
+
add_identifier
|
43
|
+
add_metadata_for_model
|
44
|
+
add_visibility
|
45
|
+
add_ingested_metadata
|
46
|
+
add_rights_statement
|
47
|
+
add_collections
|
48
|
+
add_local
|
42
49
|
|
43
|
-
|
44
|
-
keys.map { |key| key_without_numbers(key) }
|
50
|
+
self.parsed_metadata
|
45
51
|
end
|
46
52
|
|
47
|
-
def
|
48
|
-
|
53
|
+
def add_identifier
|
54
|
+
self.parsed_metadata[work_identifier] = [record[source_identifier]]
|
49
55
|
end
|
50
56
|
|
51
|
-
def
|
52
|
-
|
53
|
-
|
57
|
+
def add_metadata_for_model
|
58
|
+
if factory_class == Collection
|
59
|
+
add_collection_type_gid
|
60
|
+
else
|
61
|
+
add_file unless importerexporter.metadata_only?
|
62
|
+
add_admin_set_id
|
63
|
+
end
|
64
|
+
end
|
54
65
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
66
|
+
def add_ingested_metadata
|
67
|
+
ActiveSupport::Deprecation.warn(
|
68
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
69
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
70
|
+
)
|
71
|
+
record.sort.each do |key, value|
|
72
|
+
next if self.parser.collection_field_mapping.to_s == key_without_numbers(key)
|
59
73
|
|
60
74
|
index = key[/\d+/].to_i - 1 if key[/\d+/].to_i != 0
|
61
75
|
add_metadata(key_without_numbers(key), value, index)
|
62
76
|
end
|
63
|
-
add_file
|
64
|
-
add_visibility
|
65
|
-
add_rights_statement
|
66
|
-
add_admin_set_id
|
67
|
-
add_collections
|
68
|
-
add_local
|
69
|
-
self.parsed_metadata
|
70
77
|
end
|
71
78
|
|
72
79
|
def add_file
|
@@ -86,10 +93,20 @@ module Bulkrax
|
|
86
93
|
self.parsed_metadata[source_identifier] = hyrax_record.send(work_identifier)
|
87
94
|
self.parsed_metadata['model'] = hyrax_record.has_model.first
|
88
95
|
build_mapping_metadata
|
89
|
-
|
90
|
-
|
91
|
-
|
96
|
+
|
97
|
+
# TODO: fix the "send" parameter in the conditional below
|
98
|
+
# currently it returns: "NoMethodError - undefined method 'bulkrax_identifier' for #<Collection:0x00007fbe6a3b4248>"
|
99
|
+
if mapping['collection']&.[]('join')
|
100
|
+
self.parsed_metadata['collection'] = hyrax_record.member_of_collection_ids.join('; ')
|
101
|
+
# self.parsed_metadata['collection'] = hyrax_record.member_of_collections.map { |c| c.send(work_identifier)&.first }.compact.uniq.join(';')
|
102
|
+
else
|
103
|
+
hyrax_record.member_of_collections.each_with_index do |collection, i|
|
104
|
+
self.parsed_metadata["collection_#{i + 1}"] = collection.id
|
105
|
+
# self.parsed_metadata["collection_#{i + 1}"] = collection.send(work_identifier)&.first
|
106
|
+
end
|
92
107
|
end
|
108
|
+
|
109
|
+
build_files unless hyrax_record.is_a?(Collection)
|
93
110
|
self.parsed_metadata
|
94
111
|
end
|
95
112
|
|
@@ -97,24 +114,51 @@ module Bulkrax
|
|
97
114
|
mapping.each do |key, value|
|
98
115
|
next if Bulkrax.reserved_properties.include?(key) && !field_supported?(key)
|
99
116
|
next if key == "model"
|
117
|
+
next if value['excluded']
|
100
118
|
|
101
119
|
object_key = key if value.key?('object')
|
102
120
|
next unless hyrax_record.respond_to?(key.to_s) || object_key.present?
|
103
121
|
|
104
|
-
data = object_key.present? ? hyrax_record.send(value['object']) : hyrax_record.send(key.to_s)
|
105
122
|
if object_key.present?
|
106
|
-
|
107
|
-
|
123
|
+
build_object(value)
|
124
|
+
else
|
125
|
+
build_value(key, value)
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def build_object(value)
|
131
|
+
data = hyrax_record.send(value['object'])
|
132
|
+
return if data.empty?
|
133
|
+
|
134
|
+
data = data.to_a if data.is_a?(ActiveTriples::Relation)
|
135
|
+
object_metadata(Array.wrap(data))
|
136
|
+
end
|
108
137
|
|
109
|
-
|
110
|
-
|
111
|
-
|
138
|
+
def build_value(key, value)
|
139
|
+
data = hyrax_record.send(key.to_s)
|
140
|
+
if data.is_a?(ActiveTriples::Relation)
|
141
|
+
if value['join']
|
142
|
+
self.parsed_metadata[key_for_export(key)] = data.map { |d| prepare_export_data(d) }.join('; ').to_s
|
112
143
|
else
|
113
|
-
|
144
|
+
data.each_with_index do |d, i|
|
145
|
+
self.parsed_metadata["#{key_for_export(key)}_#{i + 1}"] = prepare_export_data(d)
|
146
|
+
end
|
114
147
|
end
|
148
|
+
else
|
149
|
+
self.parsed_metadata[key_for_export(key)] = prepare_export_data(data)
|
115
150
|
end
|
116
151
|
end
|
117
152
|
|
153
|
+
# On export the key becomes the from and the from becomes the destination. It is the opposite of the import because we are moving data the opposite direction
|
154
|
+
# metadata that does not have a specific Bulkrax entry is mapped to the key name, as matching keys coming in are mapped by the csv parser automatically
|
155
|
+
def key_for_export(key)
|
156
|
+
clean_key = key_without_numbers(key)
|
157
|
+
unnumbered_key = mapping[clean_key] ? mapping[clean_key]['from'].first : clean_key
|
158
|
+
# Bring the number back if there is one
|
159
|
+
"#{unnumbered_key}#{key.sub(clean_key, '')}"
|
160
|
+
end
|
161
|
+
|
118
162
|
def prepare_export_data(datum)
|
119
163
|
if datum.is_a?(ActiveTriples::Resource)
|
120
164
|
datum.to_uri.to_s
|
@@ -123,30 +167,34 @@ module Bulkrax
|
|
123
167
|
end
|
124
168
|
end
|
125
169
|
|
126
|
-
def object_metadata(data
|
127
|
-
data =
|
170
|
+
def object_metadata(data)
|
171
|
+
data = data.map { |d| eval(d) }.flatten # rubocop:disable Security/Eval
|
128
172
|
|
129
173
|
data.each_with_index do |obj, index|
|
130
|
-
next
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
obj
|
135
|
-
|
174
|
+
next if obj.nil?
|
175
|
+
# allow the object_key to be valid whether it's a string or symbol
|
176
|
+
obj = obj.with_indifferent_access
|
177
|
+
|
178
|
+
obj.each_key do |key|
|
179
|
+
if obj[key].is_a?(Array)
|
180
|
+
obj[key].each_with_index do |_nested_item, nested_index|
|
181
|
+
self.parsed_metadata["#{key_for_export(key)}_#{index + 1}_#{nested_index + 1}"] = prepare_export_data(obj[key][nested_index])
|
182
|
+
end
|
183
|
+
else
|
184
|
+
self.parsed_metadata["#{key_for_export(key)}_#{index + 1}"] = prepare_export_data(obj[key])
|
185
|
+
end
|
136
186
|
end
|
137
187
|
end
|
138
188
|
end
|
139
189
|
|
140
|
-
def
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
return gsub_data.map { |d| JSON.parse(d) }
|
190
|
+
def build_files
|
191
|
+
if mapping['file']&.[]('join')
|
192
|
+
self.parsed_metadata['file'] = hyrax_record.file_sets.map { |fs| filename(fs).to_s if filename(fs).present? }.compact.join('; ')
|
193
|
+
else
|
194
|
+
hyrax_record.file_sets.each_with_index do |fs, i|
|
195
|
+
self.parsed_metadata["file_#{i + 1}"] = filename(fs).to_s if filename(fs).present?
|
196
|
+
end
|
197
|
+
end
|
150
198
|
end
|
151
199
|
|
152
200
|
# In order for the existing exported hyrax_record, to be updated by a re-import
|
@@ -167,18 +215,28 @@ module Bulkrax
|
|
167
215
|
Bulkrax::CsvMatcher
|
168
216
|
end
|
169
217
|
|
218
|
+
def possible_collection_ids
|
219
|
+
ActiveSupport::Deprecation.warn(
|
220
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
221
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
222
|
+
)
|
223
|
+
@possible_collection_ids ||= record.inject([]) do |memo, (key, value)|
|
224
|
+
memo += value.split(/\s*[:;|]\s*/) if self.class.collection_field.to_s == key_without_numbers(key) && value.present?
|
225
|
+
memo
|
226
|
+
end || []
|
227
|
+
end
|
228
|
+
|
170
229
|
def collections_created?
|
171
|
-
|
172
|
-
record[self.class.collection_field].split(/\s*[:;|]\s*/).length == self.collection_ids.length
|
230
|
+
possible_collection_ids.length == self.collection_ids.length
|
173
231
|
end
|
174
232
|
|
175
|
-
def
|
233
|
+
def find_collection_ids
|
176
234
|
return self.collection_ids if collections_created?
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
self.collection_ids << c.id unless
|
235
|
+
if possible_collection_ids.present?
|
236
|
+
possible_collection_ids.each do |collection_id|
|
237
|
+
c = find_collection(collection_id)
|
238
|
+
skip = c.blank? || self.collection_ids.include?(c.id)
|
239
|
+
self.collection_ids << c.id unless skip
|
182
240
|
end
|
183
241
|
end
|
184
242
|
self.collection_ids
|
data/app/models/bulkrax/entry.rb
CHANGED
@@ -23,12 +23,22 @@ module Bulkrax
|
|
23
23
|
|
24
24
|
attr_accessor :all_attrs
|
25
25
|
|
26
|
-
delegate :parser,
|
26
|
+
delegate :parser,
|
27
|
+
:mapping,
|
28
|
+
:replace_files,
|
29
|
+
:update_files,
|
30
|
+
:keys_without_numbers,
|
31
|
+
:key_without_numbers,
|
32
|
+
to: :importerexporter
|
27
33
|
|
28
34
|
delegate :client,
|
29
|
-
|
30
|
-
|
31
|
-
|
35
|
+
:collection_name,
|
36
|
+
:user,
|
37
|
+
:related_parents_raw_mapping,
|
38
|
+
:related_parents_parsed_mapping,
|
39
|
+
:related_children_raw_mapping,
|
40
|
+
:related_children_parsed_mapping,
|
41
|
+
to: :parser
|
32
42
|
|
33
43
|
# Retrieve fields from the file
|
34
44
|
# @param data - the source data
|
@@ -61,13 +71,13 @@ module Bulkrax
|
|
61
71
|
end
|
62
72
|
|
63
73
|
def self.collection_field
|
74
|
+
ActiveSupport::Deprecation.warn(
|
75
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
76
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
77
|
+
)
|
64
78
|
Bulkrax.collection_field_mapping[self.to_s]
|
65
79
|
end
|
66
80
|
|
67
|
-
def self.children_field
|
68
|
-
Bulkrax.parent_child_field_mapping[self.to_s]
|
69
|
-
end
|
70
|
-
|
71
81
|
def build
|
72
82
|
return if type.nil?
|
73
83
|
self.save if self.new_record? # must be saved for statuses
|
@@ -96,6 +106,7 @@ module Bulkrax
|
|
96
106
|
end
|
97
107
|
|
98
108
|
def find_collection(collection_identifier)
|
109
|
+
return unless Collection.properties.keys.include?(work_identifier)
|
99
110
|
Collection.where(
|
100
111
|
work_identifier => collection_identifier
|
101
112
|
).detect { |m| m.send(work_identifier).include?(collection_identifier) }
|
@@ -14,7 +14,7 @@ module Bulkrax
|
|
14
14
|
validates :name, presence: true
|
15
15
|
validates :parser_klass, presence: true
|
16
16
|
|
17
|
-
delegate :write, :create_from_collection, :create_from_importer, :create_from_worktype, to: :parser
|
17
|
+
delegate :write, :create_from_collection, :create_from_importer, :create_from_worktype, :create_from_all, to: :parser
|
18
18
|
|
19
19
|
def export
|
20
20
|
current_run && setup_export_path
|
@@ -25,6 +25,8 @@ module Bulkrax
|
|
25
25
|
create_from_importer
|
26
26
|
when 'worktype'
|
27
27
|
create_from_worktype
|
28
|
+
when 'all'
|
29
|
+
create_from_all
|
28
30
|
end
|
29
31
|
rescue StandardError => e
|
30
32
|
status_info(e)
|
@@ -77,7 +79,8 @@ module Bulkrax
|
|
77
79
|
[
|
78
80
|
[I18n.t('bulkrax.exporter.labels.importer'), 'importer'],
|
79
81
|
[I18n.t('bulkrax.exporter.labels.collection'), 'collection'],
|
80
|
-
[I18n.t('bulkrax.exporter.labels.worktype'), 'worktype']
|
82
|
+
[I18n.t('bulkrax.exporter.labels.worktype'), 'worktype'],
|
83
|
+
[I18n.t('bulkrax.exporter.labels.all'), 'all']
|
81
84
|
]
|
82
85
|
end
|
83
86
|
|
@@ -105,18 +108,22 @@ module Bulkrax
|
|
105
108
|
end
|
106
109
|
|
107
110
|
def exporter_export_path
|
108
|
-
@exporter_export_path ||= File.join(
|
111
|
+
@exporter_export_path ||= File.join(parser.base_path('export'), self.id.to_s, self.exporter_runs.last.id.to_s)
|
109
112
|
end
|
110
113
|
|
111
114
|
def exporter_export_zip_path
|
112
|
-
@exporter_export_zip_path ||= File.join(
|
115
|
+
@exporter_export_zip_path ||= File.join(parser.base_path('export'), "export_#{self.id}_#{self.exporter_runs.last.id}.zip")
|
113
116
|
rescue
|
114
|
-
@exporter_export_zip_path ||= File.join(
|
117
|
+
@exporter_export_zip_path ||= File.join(parser.base_path('export'), "export_#{self.id}_0.zip")
|
115
118
|
end
|
116
119
|
|
117
120
|
def export_properties
|
118
121
|
properties = Hyrax.config.registered_curation_concern_types.map { |work| work.constantize.properties.keys }.flatten.uniq.sort
|
119
122
|
properties.reject { |prop| Bulkrax.reserved_properties.include?(prop) }
|
120
123
|
end
|
124
|
+
|
125
|
+
def metadata_only?
|
126
|
+
export_type == 'metadata'
|
127
|
+
end
|
121
128
|
end
|
122
129
|
end
|
@@ -18,8 +18,7 @@ module Bulkrax
|
|
18
18
|
validates :admin_set_id, presence: true
|
19
19
|
validates :parser_klass, presence: true
|
20
20
|
|
21
|
-
delegate :valid_import?, :
|
22
|
-
:write_errored_entries_file, :visibility, to: :parser
|
21
|
+
delegate :valid_import?, :write_errored_entries_file, :visibility, to: :parser
|
23
22
|
|
24
23
|
attr_accessor :only_updates, :file_style, :file
|
25
24
|
attr_writer :current_run
|
@@ -97,7 +96,11 @@ module Bulkrax
|
|
97
96
|
end
|
98
97
|
|
99
98
|
def current_run
|
100
|
-
@current_run ||=
|
99
|
+
@current_run ||= if file? && zip?
|
100
|
+
self.importer_runs.create!
|
101
|
+
else
|
102
|
+
self.importer_runs.create!(total_work_entries: self.limit || parser.works_total, total_collection_entries: parser.collections_total)
|
103
|
+
end
|
101
104
|
end
|
102
105
|
|
103
106
|
def last_run
|
@@ -149,11 +152,11 @@ module Bulkrax
|
|
149
152
|
|
150
153
|
# If the import data is zipped, unzip it to this path
|
151
154
|
def importer_unzip_path
|
152
|
-
@importer_unzip_path ||= File.join(
|
155
|
+
@importer_unzip_path ||= File.join(parser.base_path, "import_#{path_string}")
|
153
156
|
end
|
154
157
|
|
155
158
|
def errored_entries_csv_path
|
156
|
-
@errored_entries_csv_path ||= File.join(
|
159
|
+
@errored_entries_csv_path ||= File.join(parser.base_path, "import_#{path_string}_errored_entries.csv")
|
157
160
|
end
|
158
161
|
|
159
162
|
def path_string
|
@@ -161,5 +164,9 @@ module Bulkrax
|
|
161
164
|
rescue
|
162
165
|
"#{self.id}_#{self.created_at.strftime('%Y%m%d%H%M%S')}"
|
163
166
|
end
|
167
|
+
|
168
|
+
def metadata_only?
|
169
|
+
parser.parser_fields['metadata_only'] == true
|
170
|
+
end
|
164
171
|
end
|
165
172
|
end
|
@@ -26,6 +26,10 @@ module Bulkrax
|
|
26
26
|
end
|
27
27
|
|
28
28
|
def build_metadata
|
29
|
+
ActiveSupport::Deprecation.warn(
|
30
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
31
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
32
|
+
)
|
29
33
|
self.parsed_metadata = {}
|
30
34
|
self.parsed_metadata[work_identifier] = [record.header.identifier]
|
31
35
|
|
@@ -56,7 +60,7 @@ module Bulkrax
|
|
56
60
|
# Retrieve list of collections for the entry; add to collection_ids
|
57
61
|
# If OAI-PMH doesn't return setSpec in the headers for GetRecord, use parser.collection_name
|
58
62
|
# in this case, if 'All' is selected, records will not be added to a collection.
|
59
|
-
def
|
63
|
+
def find_collection_ids
|
60
64
|
return self.collection_ids if collections_created?
|
61
65
|
if sets.blank? || parser.collection_name != 'all'
|
62
66
|
# c = Collection.where(Bulkrax.system_identifier_field => importerexporter.unique_collection_identifier(parser.collection_name)).first
|
@@ -14,6 +14,10 @@ module Bulkrax
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def self.data_for_entry(data, source_id)
|
17
|
+
ActiveSupport::Deprecation.warn(
|
18
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
19
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
20
|
+
)
|
17
21
|
reader = data
|
18
22
|
format = reader.class.format.to_sym
|
19
23
|
collections = []
|
@@ -22,7 +26,7 @@ module Bulkrax
|
|
22
26
|
data = RDF::Writer.for(format).buffer do |writer|
|
23
27
|
reader.each_statement do |statement|
|
24
28
|
collections << statement.object.to_s if collection_field.present? && collection_field == statement.predicate.to_s
|
25
|
-
children << statement.object.to_s if
|
29
|
+
children << statement.object.to_s if related_children_parsed_mapping.present? && related_children_parsed_mapping == statement.predicate.to_s
|
26
30
|
delete = statement.object.to_s if /deleted/.match?(statement.predicate.to_s)
|
27
31
|
writer << statement
|
28
32
|
end
|
@@ -37,12 +41,13 @@ module Bulkrax
|
|
37
41
|
}
|
38
42
|
end
|
39
43
|
|
40
|
-
def self.
|
41
|
-
|
42
|
-
|
44
|
+
def self.related_children_parsed_mapping
|
45
|
+
return @related_children_parsed_mapping if @related_children_parsed_mapping.present?
|
46
|
+
|
47
|
+
rdf_related_children_field_mapping = Bulkrax.field_mappings['Bulkrax::RdfParser']&.select { |_, h| h.key?('related_children_field_mapping') }
|
48
|
+
return if rdf_related_children_field_mapping.blank?
|
43
49
|
|
44
|
-
|
45
|
-
Bulkrax.parent_child_field_mapping[self.to_s]
|
50
|
+
@related_children_parsed_mapping = rdf_related_children_field_mapping&.keys&.first
|
46
51
|
end
|
47
52
|
|
48
53
|
def record
|
@@ -50,6 +55,10 @@ module Bulkrax
|
|
50
55
|
end
|
51
56
|
|
52
57
|
def build_metadata
|
58
|
+
ActiveSupport::Deprecation.warn(
|
59
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
60
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
61
|
+
)
|
53
62
|
raise StandardError, 'Record not found' if record.nil?
|
54
63
|
raise StandardError, "Missing source identifier (#{source_identifier})" if self.raw_metadata[source_identifier].blank?
|
55
64
|
|
@@ -76,7 +85,7 @@ module Bulkrax
|
|
76
85
|
self.raw_metadata['collection'].length == self.collection_ids.length
|
77
86
|
end
|
78
87
|
|
79
|
-
def
|
88
|
+
def find_collection_ids
|
80
89
|
return self.collection_ids if collections_created?
|
81
90
|
if self.raw_metadata['collection'].present?
|
82
91
|
self.raw_metadata['collection'].each do |collection|
|
@@ -39,6 +39,10 @@ module Bulkrax
|
|
39
39
|
end
|
40
40
|
|
41
41
|
def build_metadata
|
42
|
+
ActiveSupport::Deprecation.warn(
|
43
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
44
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
45
|
+
)
|
42
46
|
raise StandardError, 'Record not found' if record.nil?
|
43
47
|
raise StandardError, "Missing source identifier (#{source_identifier})" if self.raw_metadata[source_identifier].blank?
|
44
48
|
self.parsed_metadata = {}
|
@@ -40,13 +40,13 @@ module Bulkrax
|
|
40
40
|
end
|
41
41
|
end
|
42
42
|
|
43
|
-
#
|
43
|
+
# Prepend the file_set id to ensure a unique filename
|
44
44
|
def filename(file_set)
|
45
45
|
return if file_set.original_file.blank?
|
46
46
|
fn = file_set.original_file.file_name.first
|
47
47
|
mime = Mime::Type.lookup(file_set.original_file.mime_type)
|
48
48
|
ext_mime = MIME::Types.of(file_set.original_file.file_name).first
|
49
|
-
if fn.include?(file_set.id)
|
49
|
+
if fn.include?(file_set.id) || importerexporter.metadata_only?
|
50
50
|
return fn if mime.to_s == ext_mime.to_s
|
51
51
|
return "#{fn}.#{mime.to_sym}"
|
52
52
|
else
|
@@ -43,7 +43,6 @@ module Bulkrax
|
|
43
43
|
|
44
44
|
value = if matcher
|
45
45
|
result = matcher.result(self, node_content)
|
46
|
-
next unless result
|
47
46
|
matched_metadata(multiple, name, result, object_multiple)
|
48
47
|
elsif multiple
|
49
48
|
Rails.logger.info("Bulkrax Column automatically matched #{node_name}, #{node_content}")
|
@@ -53,12 +52,20 @@ module Bulkrax
|
|
53
52
|
single_metadata(node_content)
|
54
53
|
end
|
55
54
|
|
56
|
-
|
55
|
+
object_name.present? ? set_parsed_object_data(object_multiple, object_name, name, index, value) : set_parsed_data(name, value)
|
57
56
|
end
|
58
57
|
end
|
59
58
|
|
60
|
-
def set_parsed_data(
|
59
|
+
def set_parsed_data(name, value)
|
60
|
+
return parsed_metadata[name] = value unless multiple?(name)
|
61
|
+
|
62
|
+
parsed_metadata[name] ||= []
|
63
|
+
parsed_metadata[name] += Array.wrap(value).flatten
|
64
|
+
end
|
65
|
+
|
66
|
+
def set_parsed_object_data(object_multiple, object_name, name, index, value)
|
61
67
|
if object_multiple
|
68
|
+
index ||= 0
|
62
69
|
parsed_metadata[object_name][index] ||= {}
|
63
70
|
parsed_metadata[object_name][index][name] ||= []
|
64
71
|
if value.is_a?(Array)
|
@@ -66,20 +73,13 @@ module Bulkrax
|
|
66
73
|
else
|
67
74
|
parsed_metadata[object_name][index][name] = value
|
68
75
|
end
|
69
|
-
|
76
|
+
else
|
70
77
|
parsed_metadata[object_name][name] ||= []
|
71
78
|
if value.is_a?(Array)
|
72
79
|
parsed_metadata[object_name][name] += value
|
73
80
|
else
|
74
81
|
parsed_metadata[object_name][name] = value
|
75
82
|
end
|
76
|
-
else
|
77
|
-
parsed_metadata[name] ||= []
|
78
|
-
if value.is_a?(Array)
|
79
|
-
parsed_metadata[name] += value
|
80
|
-
else
|
81
|
-
parsed_metadata[name] = value
|
82
|
-
end
|
83
83
|
end
|
84
84
|
end
|
85
85
|
|
@@ -124,12 +124,43 @@ module Bulkrax
|
|
124
124
|
field = field.gsub('_attributes', '')
|
125
125
|
|
126
126
|
return false if excluded?(field)
|
127
|
-
return true if
|
127
|
+
return true if supported_bulkrax_fields.include?(field)
|
128
128
|
return factory_class.method_defined?(field) && factory_class.properties[field].present?
|
129
129
|
end
|
130
130
|
|
131
|
+
def supported_bulkrax_fields
|
132
|
+
ActiveSupport::Deprecation.warn(
|
133
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
134
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
135
|
+
)
|
136
|
+
@supported_bulkrax_fields ||=
|
137
|
+
%W[
|
138
|
+
id
|
139
|
+
file
|
140
|
+
remote_files
|
141
|
+
model
|
142
|
+
delete
|
143
|
+
#{parser.collection_field_mapping}
|
144
|
+
#{related_parents_parsed_mapping}
|
145
|
+
#{related_children_parsed_mapping}
|
146
|
+
]
|
147
|
+
end
|
148
|
+
|
131
149
|
def multiple?(field)
|
132
|
-
|
150
|
+
ActiveSupport::Deprecation.warn(
|
151
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
152
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
153
|
+
)
|
154
|
+
@multiple_bulkrax_fields ||=
|
155
|
+
%W[
|
156
|
+
file
|
157
|
+
remote_files
|
158
|
+
#{parser.collection_field_mapping}
|
159
|
+
#{related_parents_parsed_mapping}
|
160
|
+
#{related_children_parsed_mapping}
|
161
|
+
]
|
162
|
+
|
163
|
+
return true if @multiple_bulkrax_fields.include?(field)
|
133
164
|
return false if field == 'model'
|
134
165
|
|
135
166
|
field_supported?(field) && factory_class&.properties&.[](field)&.[]('multiple')
|