bulkrax 1.0.2 → 2.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/app/controllers/bulkrax/exporters_controller.rb +12 -4
- data/app/controllers/bulkrax/importers_controller.rb +23 -17
- data/app/factories/bulkrax/object_factory.rb +84 -63
- data/app/jobs/bulkrax/create_relationships_job.rb +156 -0
- data/app/jobs/bulkrax/delete_work_job.rb +6 -2
- data/app/jobs/bulkrax/export_work_job.rb +3 -1
- data/app/jobs/bulkrax/exporter_job.rb +1 -0
- data/app/jobs/bulkrax/{import_work_collection_job.rb → import_collection_job.rb} +4 -2
- data/app/jobs/bulkrax/import_file_set_job.rb +69 -0
- data/app/jobs/bulkrax/import_work_job.rb +2 -0
- data/app/jobs/bulkrax/importer_job.rb +18 -1
- data/app/matchers/bulkrax/application_matcher.rb +5 -5
- data/app/models/bulkrax/csv_collection_entry.rb +8 -6
- data/app/models/bulkrax/csv_entry.rb +132 -65
- data/app/models/bulkrax/csv_file_set_entry.rb +26 -0
- data/app/models/bulkrax/entry.rb +19 -8
- data/app/models/bulkrax/exporter.rb +12 -5
- data/app/models/bulkrax/importer.rb +24 -5
- data/app/models/bulkrax/oai_entry.rb +5 -1
- data/app/models/bulkrax/rdf_entry.rb +16 -7
- data/app/models/bulkrax/xml_entry.rb +4 -0
- data/app/models/concerns/bulkrax/dynamic_record_lookup.rb +39 -0
- data/app/models/concerns/bulkrax/export_behavior.rb +2 -2
- data/app/models/concerns/bulkrax/has_matchers.rb +44 -13
- data/app/models/concerns/bulkrax/import_behavior.rb +40 -5
- data/app/models/concerns/bulkrax/importer_exporter_behavior.rb +23 -2
- data/app/models/concerns/bulkrax/status_info.rb +4 -4
- data/app/parsers/bulkrax/application_parser.rb +67 -84
- data/app/parsers/bulkrax/bagit_parser.rb +13 -4
- data/app/parsers/bulkrax/csv_parser.rb +170 -64
- data/app/parsers/bulkrax/oai_dc_parser.rb +6 -3
- data/app/parsers/bulkrax/xml_parser.rb +5 -0
- data/app/views/bulkrax/exporters/_form.html.erb +1 -1
- data/app/views/bulkrax/exporters/show.html.erb +2 -1
- data/app/views/bulkrax/importers/index.html.erb +17 -17
- data/app/views/bulkrax/importers/show.html.erb +52 -6
- data/config/locales/bulkrax.en.yml +1 -0
- data/db/migrate/20190731114016_change_importer_and_exporter_to_polymorphic.rb +5 -1
- data/db/migrate/20211004170708_change_bulkrax_statuses_error_message_column_type_to_text.rb +5 -0
- data/db/migrate/20211203195233_rename_children_counters_to_relationships.rb +6 -0
- data/db/migrate/20211220195027_add_file_set_counters_to_importer_runs.rb +7 -0
- data/db/migrate/20220118001339_add_import_attempts_to_entries.rb +5 -0
- data/db/migrate/20220119213325_add_work_counters_to_importer_runs.rb +6 -0
- data/lib/bulkrax/engine.rb +1 -1
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +9 -17
- data/lib/generators/bulkrax/templates/bin/importer +17 -11
- data/lib/generators/bulkrax/templates/config/bulkrax_api.yml +3 -1
- data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb +7 -12
- metadata +22 -10
- data/app/jobs/bulkrax/child_relationships_job.rb +0 -128
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5a5ca12cf97da052d9855404c783d52e41679587332ac72aac25d85237cf8231
|
4
|
+
data.tar.gz: 449209843d001cdac64dd4073db130920bb62a3193b2eefd53b94f2ec15f6a08
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 296ef3a7a7af8765aaae12353bec7ecfb35a97546955520ac544269525adfe459f3e686886fefbff54c91ba09f7da20c3b6b8c62bcb3fc7be6970f7967defa17
|
7
|
+
data.tar.gz: e7a13f384940a89d42c5efb5d52b8e940939c103dceb35890e08cb56ee020c2c664d1bb4d57945ae9d7afb2acd822385861e004ab7d69c776a0816d688cf66e1
|
data/README.md
CHANGED
@@ -51,9 +51,12 @@ module Bulkrax
|
|
51
51
|
if @exporter.save
|
52
52
|
if params[:commit] == 'Create and Export'
|
53
53
|
# Use perform now for export
|
54
|
-
Bulkrax::ExporterJob.
|
54
|
+
Bulkrax::ExporterJob.perform_later(@exporter.id)
|
55
|
+
message = 'Exporter was successfully created. A download link will appear once it completes.'
|
56
|
+
else
|
57
|
+
message = 'Exporter was successfully created.'
|
55
58
|
end
|
56
|
-
redirect_to exporters_path, notice:
|
59
|
+
redirect_to exporters_path, notice: message
|
57
60
|
else
|
58
61
|
render :new
|
59
62
|
end
|
@@ -63,8 +66,13 @@ module Bulkrax
|
|
63
66
|
def update
|
64
67
|
field_mapping_params
|
65
68
|
if @exporter.update(exporter_params)
|
66
|
-
|
67
|
-
|
69
|
+
if params[:commit] == 'Update and Re-Export All Items'
|
70
|
+
Bulkrax::ExporterJob.perform_later(@exporter.id)
|
71
|
+
message = 'Exporter was successfully updated. A download link will appear once it completes.'
|
72
|
+
else
|
73
|
+
'Exporter was successfully updated.'
|
74
|
+
end
|
75
|
+
redirect_to exporters_path, notice: message
|
68
76
|
else
|
69
77
|
render :edit
|
70
78
|
end
|
@@ -37,6 +37,7 @@ module Bulkrax
|
|
37
37
|
|
38
38
|
@work_entries = @importer.entries.where(type: @importer.parser.entry_class.to_s).page(params[:work_entries_page]).per(30)
|
39
39
|
@collection_entries = @importer.entries.where(type: @importer.parser.collection_entry_class.to_s).page(params[:collections_entries_page]).per(30)
|
40
|
+
@file_set_entries = @importer.entries.where(type: @importer.parser.file_set_entry_class.to_s).page(params[:file_set_entries_page]).per(30)
|
40
41
|
end
|
41
42
|
end
|
42
43
|
|
@@ -113,23 +114,9 @@ module Bulkrax
|
|
113
114
|
if @importer.update(importer_params)
|
114
115
|
files_for_import(file, cloud_files) unless file.nil? && cloud_files.nil?
|
115
116
|
# do not perform the import
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
elsif params[:commit] == 'Update and Harvest Updated Items'
|
120
|
-
Bulkrax::ImporterJob.perform_later(@importer.id, true)
|
121
|
-
elsif params[:commit] == 'Update Metadata and Files'
|
122
|
-
@importer.parser_fields['update_files'] = true
|
123
|
-
@importer.save
|
124
|
-
Bulkrax::ImporterJob.perform_later(@importer.id)
|
125
|
-
# Perform a full metadata and files re-import; do the same for an OAI re-harvest of all items
|
126
|
-
elsif params[:commit] == ('Update and Replace Files' || 'Update and Re-Harvest All Items')
|
127
|
-
@importer.parser_fields['replace_files'] = true
|
128
|
-
@importer.save
|
129
|
-
Bulkrax::ImporterJob.perform_later(@importer.id)
|
130
|
-
# In all other cases, perform a metadata-only re-import
|
131
|
-
else
|
132
|
-
Bulkrax::ImporterJob.perform_later(@importer.id)
|
117
|
+
unless params[:commit] == 'Update Importer'
|
118
|
+
set_files_parser_fields
|
119
|
+
Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id, update_harvest)
|
133
120
|
end
|
134
121
|
if api_request?
|
135
122
|
json_response('updated', :ok, 'Importer was successfully updated.')
|
@@ -310,6 +297,25 @@ module Bulkrax
|
|
310
297
|
redirect_to path, notice: message
|
311
298
|
end
|
312
299
|
end
|
300
|
+
|
301
|
+
# update methods (for commit deciphering)
|
302
|
+
def update_harvest
|
303
|
+
# OAI-only - selective re-harvest
|
304
|
+
params[:commit] == 'Update and Harvest Updated Items'
|
305
|
+
end
|
306
|
+
|
307
|
+
def set_files_parser_fields
|
308
|
+
if params[:commit] == 'Update Metadata and Files'
|
309
|
+
@importer.parser_fields['update_files'] = true
|
310
|
+
elsif params[:commit] == ('Update and Replace Files' || 'Update and Re-Harvest All Items')
|
311
|
+
@importer.parser_fields['replace_files'] = true
|
312
|
+
elsif params[:commit] == 'Update and Harvest Updated Items'
|
313
|
+
return
|
314
|
+
else
|
315
|
+
@importer.parser_fields['metadata_only'] = true
|
316
|
+
end
|
317
|
+
@importer.save
|
318
|
+
end
|
313
319
|
end
|
314
320
|
# rubocop:enable Metrics/ClassLength
|
315
321
|
end
|
@@ -4,16 +4,24 @@ module Bulkrax
|
|
4
4
|
class ObjectFactory
|
5
5
|
extend ActiveModel::Callbacks
|
6
6
|
include Bulkrax::FileFactory
|
7
|
+
include DynamicRecordLookup
|
8
|
+
|
7
9
|
define_model_callbacks :save, :create
|
8
|
-
attr_reader :attributes, :object, :source_identifier_value, :klass, :replace_files, :update_files, :work_identifier
|
10
|
+
attr_reader :attributes, :object, :source_identifier_value, :klass, :replace_files, :update_files, :work_identifier, :collection_field_mapping, :related_parents_parsed_mapping
|
9
11
|
|
10
12
|
# rubocop:disable Metrics/ParameterLists
|
11
|
-
def initialize(attributes:, source_identifier_value:, work_identifier:, replace_files: false, user: nil, klass: nil, update_files: false)
|
13
|
+
def initialize(attributes:, source_identifier_value:, work_identifier:, collection_field_mapping:, related_parents_parsed_mapping: nil, replace_files: false, user: nil, klass: nil, update_files: false)
|
14
|
+
ActiveSupport::Deprecation.warn(
|
15
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
16
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
17
|
+
)
|
12
18
|
@attributes = ActiveSupport::HashWithIndifferentAccess.new(attributes)
|
13
19
|
@replace_files = replace_files
|
14
20
|
@update_files = update_files
|
15
21
|
@user = user || User.batch_user
|
16
22
|
@work_identifier = work_identifier
|
23
|
+
@collection_field_mapping = collection_field_mapping
|
24
|
+
@related_parents_parsed_mapping = related_parents_parsed_mapping
|
17
25
|
@source_identifier_value = source_identifier_value
|
18
26
|
@klass = klass || Bulkrax.default_work_type.constantize
|
19
27
|
end
|
@@ -28,7 +36,7 @@ module Bulkrax
|
|
28
36
|
arg_hash = { id: attributes[:id], name: 'UPDATE', klass: klass }
|
29
37
|
@object = find
|
30
38
|
if object
|
31
|
-
object.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX
|
39
|
+
object.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX if object.respond_to?(:reindex_extent)
|
32
40
|
ActiveSupport::Notifications.instrument('import.importer', arg_hash) { update }
|
33
41
|
else
|
34
42
|
ActiveSupport::Notifications.instrument('import.importer', arg_hash.merge(name: 'CREATE')) { create }
|
@@ -41,20 +49,27 @@ module Bulkrax
|
|
41
49
|
self.run
|
42
50
|
# Create the error exception if the object is not validly saved for some reason
|
43
51
|
raise ActiveFedora::RecordInvalid, object if !object.persisted? || object.changed?
|
52
|
+
object
|
44
53
|
end
|
45
54
|
|
46
55
|
def update
|
47
56
|
raise "Object doesn't exist" unless object
|
48
|
-
destroy_existing_files if @replace_files && klass
|
57
|
+
destroy_existing_files if @replace_files && ![Collection, FileSet].include?(klass)
|
49
58
|
attrs = attribute_update
|
50
59
|
run_callbacks :save do
|
51
|
-
klass == Collection
|
60
|
+
if klass == Collection
|
61
|
+
update_collection(attrs)
|
62
|
+
elsif klass == FileSet
|
63
|
+
update_file_set(attrs)
|
64
|
+
else
|
65
|
+
work_actor.update(environment(attrs))
|
66
|
+
end
|
52
67
|
end
|
53
68
|
log_updated(object)
|
54
69
|
end
|
55
70
|
|
56
71
|
def find
|
57
|
-
return find_by_id if attributes[:id]
|
72
|
+
return find_by_id if attributes[:id].present?
|
58
73
|
return search_by_identifier if attributes[work_identifier].present?
|
59
74
|
end
|
60
75
|
|
@@ -84,10 +99,16 @@ module Bulkrax
|
|
84
99
|
def create
|
85
100
|
attrs = create_attributes
|
86
101
|
@object = klass.new
|
87
|
-
object.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX
|
102
|
+
object.reindex_extent = Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX if object.respond_to?(:reindex_extent)
|
88
103
|
run_callbacks :save do
|
89
104
|
run_callbacks :create do
|
90
|
-
klass == Collection
|
105
|
+
if klass == Collection
|
106
|
+
create_collection(attrs)
|
107
|
+
elsif klass == FileSet
|
108
|
+
create_file_set(attrs)
|
109
|
+
else
|
110
|
+
work_actor.create(environment(attrs))
|
111
|
+
end
|
91
112
|
end
|
92
113
|
end
|
93
114
|
log_created(object)
|
@@ -121,52 +142,62 @@ module Bulkrax
|
|
121
142
|
end
|
122
143
|
|
123
144
|
def create_collection(attrs)
|
145
|
+
ActiveSupport::Deprecation.warn(
|
146
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
147
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
148
|
+
)
|
124
149
|
attrs = collection_type(attrs)
|
125
|
-
object
|
126
|
-
object
|
150
|
+
persist_collection_memberships(parent: object, child: find_collection(attributes[:child_collection_id])) if attributes[:child_collection_id].present?
|
151
|
+
persist_collection_memberships(parent: find_collection(attributes[collection_field_mapping]), child: object) if attributes[collection_field_mapping].present?
|
127
152
|
object.attributes = attrs
|
128
153
|
object.apply_depositor_metadata(@user)
|
129
154
|
object.save!
|
130
155
|
end
|
131
156
|
|
132
157
|
def update_collection(attrs)
|
133
|
-
|
134
|
-
|
158
|
+
ActiveSupport::Deprecation.warn(
|
159
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
160
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
161
|
+
)
|
162
|
+
persist_collection_memberships(parent: object, child: find_collection(attributes[:child_collection_id])) if attributes[:child_collection_id].present?
|
163
|
+
persist_collection_memberships(parent: find_collection(attributes[collection_field_mapping]), child: object) if attributes[collection_field_mapping].present?
|
135
164
|
object.attributes = attrs
|
136
165
|
object.save!
|
137
166
|
end
|
138
167
|
|
139
|
-
#
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
)
|
155
|
-
|
156
|
-
ms.flatten.compact.uniq
|
157
|
-
end
|
158
|
-
|
159
|
-
def member_of_collections
|
160
|
-
ms = object.member_of_collection_ids.to_a.map { |id| find_collection(id) }
|
161
|
-
[:collection, :collections].each do |atat|
|
162
|
-
next if attributes[atat].blank?
|
163
|
-
ms.concat(
|
164
|
-
Array.wrap(
|
165
|
-
find_collection(attributes[atat])
|
166
|
-
)
|
167
|
-
)
|
168
|
+
# This method is heavily inspired by Hyrax's AttachFilesToWorkJob
|
169
|
+
def create_file_set(attrs)
|
170
|
+
work = find_record(attributes[related_parents_parsed_mapping].first)
|
171
|
+
work_permissions = work.permissions.map(&:to_hash)
|
172
|
+
file_set_attrs = attrs.slice(*object.attributes.keys)
|
173
|
+
object.assign_attributes(file_set_attrs)
|
174
|
+
|
175
|
+
attrs['uploaded_files'].each do |uploaded_file_id|
|
176
|
+
uploaded_file = ::Hyrax::UploadedFile.find(uploaded_file_id)
|
177
|
+
next if uploaded_file.file_set_uri.present?
|
178
|
+
|
179
|
+
actor = ::Hyrax::Actors::FileSetActor.new(object, @user)
|
180
|
+
uploaded_file.update(file_set_uri: actor.file_set.uri)
|
181
|
+
actor.file_set.permissions_attributes = work_permissions
|
182
|
+
actor.create_metadata
|
183
|
+
actor.create_content(uploaded_file)
|
184
|
+
actor.attach_to_work(work)
|
168
185
|
end
|
169
|
-
|
186
|
+
|
187
|
+
object.save!
|
188
|
+
end
|
189
|
+
|
190
|
+
def update_file_set(attrs)
|
191
|
+
file_set_attrs = attrs.slice(*object.attributes.keys)
|
192
|
+
actor = ::Hyrax::Actors::FileSetActor.new(object, @user)
|
193
|
+
|
194
|
+
actor.update_metadata(file_set_attrs)
|
195
|
+
end
|
196
|
+
|
197
|
+
# Add child to parent's #member_collections
|
198
|
+
# Add parent to child's #member_of_collections
|
199
|
+
def persist_collection_memberships(parent:, child:)
|
200
|
+
::Hyrax::Collections::NestedCollectionPersistenceService.persist_nested_collection_for(parent: parent, child: child)
|
170
201
|
end
|
171
202
|
|
172
203
|
def find_collection(id)
|
@@ -192,32 +223,22 @@ module Bulkrax
|
|
192
223
|
# which is used by Hyrax::Actors::AddAsMemberOfCollectionsActor
|
193
224
|
def create_attributes
|
194
225
|
return transform_attributes if klass == Collection
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
end
|
201
|
-
transform_attributes.except(:collections).merge(member_of_collections_attributes: collection_ids)
|
202
|
-
else
|
203
|
-
transform_attributes
|
204
|
-
end
|
226
|
+
ActiveSupport::Deprecation.warn(
|
227
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
228
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
229
|
+
)
|
230
|
+
transform_attributes.except(:collections, :collection, collection_field_mapping)
|
205
231
|
end
|
206
232
|
|
207
233
|
# Strip out the :collection key, and add the member_of_collection_ids,
|
208
234
|
# which is used by Hyrax::Actors::AddAsMemberOfCollectionsActor
|
209
235
|
def attribute_update
|
210
236
|
return transform_attributes.except(:id) if klass == Collection
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
end
|
217
|
-
transform_attributes.except(:id).except(:collections).merge(member_of_collections_attributes: collection_ids)
|
218
|
-
else
|
219
|
-
transform_attributes.except(:id)
|
220
|
-
end
|
237
|
+
ActiveSupport::Deprecation.warn(
|
238
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
239
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
240
|
+
)
|
241
|
+
transform_attributes.except(:id, :collections, :collection, collection_field_mapping)
|
221
242
|
end
|
222
243
|
|
223
244
|
# Override if we need to map the attributes from the parser in
|
@@ -230,7 +251,7 @@ module Bulkrax
|
|
230
251
|
|
231
252
|
# Regardless of what the Parser gives us, these are the properties we are prepared to accept.
|
232
253
|
def permitted_attributes
|
233
|
-
klass.properties.keys.map(&:to_sym) + %i[id edit_users edit_groups read_groups visibility work_members_attributes admin_set_id]
|
254
|
+
klass.properties.keys.map(&:to_sym) + %i[id edit_users edit_groups read_groups visibility work_members_attributes admin_set_id member_of_collections_attributes]
|
234
255
|
end
|
235
256
|
end
|
236
257
|
end
|
@@ -0,0 +1,156 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Bulkrax
|
4
|
+
# Responsible for creating parent-child relationships between Works and Collections.
|
5
|
+
#
|
6
|
+
# Handles three kinds of relationships:
|
7
|
+
# - Work to Collection
|
8
|
+
# - Collection to Collection
|
9
|
+
# - Work to Work
|
10
|
+
#
|
11
|
+
# These can be established from either side of the relationship (i.e. from parent to child or from child to parent).
|
12
|
+
# This job only creates one relationship at a time. If a record needs multiple parents or children or both, individual
|
13
|
+
# jobs should be run for each of those relationships.
|
14
|
+
#
|
15
|
+
# NOTE: In the context of this job, "record" is used to generically refer
|
16
|
+
# to either an instance of a Work or an instance of a Collection.
|
17
|
+
# NOTE: In the context of this job, "identifier" is used to generically refer
|
18
|
+
# to either a record's ID or an Bulkrax::Entry's source_identifier.
|
19
|
+
class CreateRelationshipsJob < ApplicationJob
|
20
|
+
include DynamicRecordLookup
|
21
|
+
|
22
|
+
queue_as :import
|
23
|
+
|
24
|
+
attr_accessor :base_entry, :child_record, :parent_record, :importer_run
|
25
|
+
|
26
|
+
# @param entry_identifier [String] source_identifier of the base Bulkrax::Entry the job was triggered from (see #build_for_importer)
|
27
|
+
# @param parent_identifier [String] Work/Collection ID or Bulkrax::Entry source_identifier
|
28
|
+
# @param child_identifier [String] Work/Collection ID or Bulkrax::Entry source_identifier
|
29
|
+
# @param importer_run [Bulkrax::ImporterRun] current importer run (needed to properly update counters)
|
30
|
+
#
|
31
|
+
# The entry_identifier is used to lookup the @base_entry for the job (a.k.a. the entry the job was called from).
|
32
|
+
# The @base_entry defines the context of the relationship (e.g. "this entry (@base_entry) should have a parent").
|
33
|
+
# Whether the @base_entry is the parent or the child in the relationship is determined by the presence of a
|
34
|
+
# parent_identifier or child_identifier param. For example, if a parent_identifier is passed, we know @base_entry
|
35
|
+
# is the child in the relationship, and vice versa if a child_identifier is passed.
|
36
|
+
def perform(entry_identifier:, parent_identifier: nil, child_identifier: nil, importer_run:)
|
37
|
+
@base_entry = Entry.find_by(identifier: entry_identifier)
|
38
|
+
@importer_run = importer_run
|
39
|
+
if parent_identifier.present?
|
40
|
+
@child_record = find_record(entry_identifier)
|
41
|
+
@parent_record = find_record(parent_identifier)
|
42
|
+
elsif child_identifier.present?
|
43
|
+
@parent_record = find_record(entry_identifier)
|
44
|
+
@child_record = find_record(child_identifier)
|
45
|
+
else
|
46
|
+
raise ::StandardError, %("#{entry_identifier}" needs either a child or a parent to create a relationship)
|
47
|
+
end
|
48
|
+
|
49
|
+
if @child_record.blank? || @parent_record.blank?
|
50
|
+
reschedule(
|
51
|
+
entry_identifier: entry_identifier,
|
52
|
+
parent_identifier: parent_identifier,
|
53
|
+
child_identifier: child_identifier,
|
54
|
+
importer_run: importer_run
|
55
|
+
)
|
56
|
+
return false # stop current job from continuing to run after rescheduling
|
57
|
+
end
|
58
|
+
|
59
|
+
create_relationship
|
60
|
+
rescue ::StandardError => e
|
61
|
+
base_entry.status_info(e)
|
62
|
+
importer_run.increment!(:failed_relationships) # rubocop:disable Rails/SkipsModelValidations
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
def create_relationship
|
68
|
+
if parent_record.is_a?(::Collection) && child_record.is_a?(::Collection)
|
69
|
+
collection_parent_collection_child
|
70
|
+
elsif parent_record.is_a?(::Collection) && curation_concern?(child_record)
|
71
|
+
collection_parent_work_child
|
72
|
+
elsif curation_concern?(parent_record) && child_record.is_a?(::Collection)
|
73
|
+
raise ::StandardError, 'a Collection may not be assigned as a child of a Work'
|
74
|
+
else
|
75
|
+
work_parent_work_child
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def user
|
80
|
+
@user ||= importer_run.importer.user
|
81
|
+
end
|
82
|
+
|
83
|
+
# Work-Collection membership is added to the child as member_of_collection_ids
|
84
|
+
# This is adding the reverse relationship, from the child to the parent
|
85
|
+
def collection_parent_work_child
|
86
|
+
ActiveSupport::Deprecation.warn(
|
87
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
88
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
89
|
+
)
|
90
|
+
attrs = { id: child_record.id, member_of_collections_attributes: { 0 => { id: parent_record.id } } }
|
91
|
+
ObjectFactory.new(
|
92
|
+
attributes: attrs,
|
93
|
+
source_identifier_value: nil, # sending the :id in the attrs means the factory doesn't need a :source_identifier_value
|
94
|
+
work_identifier: base_entry.parser.work_identifier,
|
95
|
+
collection_field_mapping: base_entry.parser.collection_field_mapping,
|
96
|
+
replace_files: false,
|
97
|
+
user: user,
|
98
|
+
klass: child_record.class
|
99
|
+
).run
|
100
|
+
# TODO: add counters for :processed_parents and :failed_parents
|
101
|
+
importer_run.increment!(:processed_relationships) # rubocop:disable Rails/SkipsModelValidations
|
102
|
+
end
|
103
|
+
|
104
|
+
# Collection-Collection membership is added to the as member_ids
|
105
|
+
def collection_parent_collection_child
|
106
|
+
ActiveSupport::Deprecation.warn(
|
107
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
108
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
109
|
+
)
|
110
|
+
attrs = { id: parent_record.id, child_collection_id: child_record.id }
|
111
|
+
ObjectFactory.new(
|
112
|
+
attributes: attrs,
|
113
|
+
source_identifier_value: nil, # sending the :id in the attrs means the factory doesn't need a :source_identifier_value
|
114
|
+
work_identifier: base_entry.parser.work_identifier,
|
115
|
+
collection_field_mapping: base_entry.parser.collection_field_mapping,
|
116
|
+
replace_files: false,
|
117
|
+
user: user,
|
118
|
+
klass: parent_record.class
|
119
|
+
).run
|
120
|
+
# TODO: add counters for :processed_parents and :failed_parents
|
121
|
+
importer_run.increment!(:processed_relationships) # rubocop:disable Rails/SkipsModelValidations
|
122
|
+
end
|
123
|
+
|
124
|
+
# Work-Work membership is added to the parent as member_ids
|
125
|
+
def work_parent_work_child
|
126
|
+
ActiveSupport::Deprecation.warn(
|
127
|
+
'Creating Collections using the collection_field_mapping will no longer be supported as of Bulkrax version 3.0.' \
|
128
|
+
' Please configure Bulkrax to use related_parents_field_mapping and related_children_field_mapping instead.'
|
129
|
+
)
|
130
|
+
attrs = {
|
131
|
+
id: parent_record.id,
|
132
|
+
work_members_attributes: { 0 => { id: child_record.id } }
|
133
|
+
}
|
134
|
+
ObjectFactory.new(
|
135
|
+
attributes: attrs,
|
136
|
+
source_identifier_value: nil, # sending the :id in the attrs means the factory doesn't need a :source_identifier_value
|
137
|
+
work_identifier: base_entry.parser.work_identifier,
|
138
|
+
collection_field_mapping: base_entry.parser.collection_field_mapping,
|
139
|
+
replace_files: false,
|
140
|
+
user: user,
|
141
|
+
klass: parent_record.class
|
142
|
+
).run
|
143
|
+
# TODO: add counters for :processed_parents and :failed_parents
|
144
|
+
importer_run.increment!(:processed_relationships) # rubocop:disable Rails/SkipsModelValidations
|
145
|
+
end
|
146
|
+
|
147
|
+
def reschedule(entry_identifier:, parent_identifier:, child_identifier:, importer_run:)
|
148
|
+
CreateRelationshipsJob.set(wait: 10.minutes).perform_later(
|
149
|
+
entry_identifier: entry_identifier,
|
150
|
+
parent_identifier: parent_identifier,
|
151
|
+
child_identifier: child_identifier,
|
152
|
+
importer_run: importer_run
|
153
|
+
)
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
@@ -8,8 +8,12 @@ module Bulkrax
|
|
8
8
|
def perform(entry, importer_run)
|
9
9
|
work = entry.factory.find
|
10
10
|
work&.delete
|
11
|
-
importer_run.increment!(:deleted_records)
|
12
|
-
importer_run.decrement!(:enqueued_records)
|
11
|
+
ImporterRun.find(importer_run.id).increment!(:deleted_records)
|
12
|
+
ImporterRun.find(importer_run.id).decrement!(:enqueued_records)
|
13
|
+
entry.save!
|
14
|
+
entry.importer.current_run = ImporterRun.find(importer_run.id)
|
15
|
+
entry.importer.record_status
|
16
|
+
entry.status_info("Deleted", ImporterRun.find(importer_run.id))
|
13
17
|
end
|
14
18
|
# rubocop:enable Rails/SkipsModelValidations
|
15
19
|
end
|
@@ -26,12 +26,14 @@ module Bulkrax
|
|
26
26
|
# rubocop:enable Rails/SkipsModelValidations
|
27
27
|
end
|
28
28
|
exporter_run = ExporterRun.find(args[1])
|
29
|
-
return if exporter_run.enqueued_records.positive?
|
29
|
+
return entry if exporter_run.enqueued_records.positive?
|
30
30
|
if exporter_run.failed_records.positive?
|
31
31
|
exporter_run.exporter.status_info('Complete (with failures)')
|
32
32
|
else
|
33
33
|
exporter_run.exporter.status_info('Complete')
|
34
34
|
end
|
35
|
+
|
36
|
+
return entry
|
35
37
|
end
|
36
38
|
end
|
37
39
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module Bulkrax
|
4
|
-
class
|
4
|
+
class ImportCollectionJob < ApplicationJob
|
5
5
|
queue_as :import
|
6
6
|
|
7
7
|
# rubocop:disable Rails/SkipsModelValidations
|
@@ -11,9 +11,11 @@ module Bulkrax
|
|
11
11
|
entry.build
|
12
12
|
entry.save
|
13
13
|
add_user_to_permission_template!(entry)
|
14
|
+
ImporterRun.find(args[1]).increment!(:processed_records)
|
14
15
|
ImporterRun.find(args[1]).increment!(:processed_collections)
|
15
16
|
ImporterRun.find(args[1]).decrement!(:enqueued_records)
|
16
17
|
rescue => e
|
18
|
+
ImporterRun.find(args[1]).increment!(:failed_records)
|
17
19
|
ImporterRun.find(args[1]).increment!(:failed_collections)
|
18
20
|
ImporterRun.find(args[1]).decrement!(:enqueued_records)
|
19
21
|
raise e
|
@@ -28,7 +30,7 @@ module Bulkrax
|
|
28
30
|
collection = entry.factory.find
|
29
31
|
permission_template = Hyrax::PermissionTemplate.find_or_create_by!(source_id: collection.id)
|
30
32
|
|
31
|
-
Hyrax::PermissionTemplateAccess.
|
33
|
+
Hyrax::PermissionTemplateAccess.find_or_create_by!(
|
32
34
|
permission_template_id: permission_template.id,
|
33
35
|
agent_id: user.user_key,
|
34
36
|
agent_type: 'user',
|
@@ -0,0 +1,69 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Bulkrax
|
4
|
+
class MissingParentError < ::StandardError; end
|
5
|
+
class ImportFileSetJob < ApplicationJob
|
6
|
+
include DynamicRecordLookup
|
7
|
+
|
8
|
+
queue_as :import
|
9
|
+
|
10
|
+
def perform(entry_id, importer_run_id)
|
11
|
+
entry = Entry.find(entry_id)
|
12
|
+
parent_identifier = entry.raw_metadata[entry.related_parents_raw_mapping]&.strip
|
13
|
+
|
14
|
+
validate_parent!(parent_identifier)
|
15
|
+
|
16
|
+
entry.build
|
17
|
+
if entry.succeeded?
|
18
|
+
# rubocop:disable Rails/SkipsModelValidations
|
19
|
+
ImporterRun.find(importer_run_id).increment!(:processed_records)
|
20
|
+
ImporterRun.find(importer_run_id).increment!(:processed_file_sets)
|
21
|
+
else
|
22
|
+
ImporterRun.find(importer_run_id).increment!(:failed_records)
|
23
|
+
ImporterRun.find(importer_run_id).increment!(:failed_file_sets)
|
24
|
+
# rubocop:enable Rails/SkipsModelValidations
|
25
|
+
end
|
26
|
+
ImporterRun.find(importer_run_id).decrement!(:enqueued_records) # rubocop:disable Rails/SkipsModelValidations
|
27
|
+
entry.save!
|
28
|
+
|
29
|
+
rescue MissingParentError => e
|
30
|
+
# try waiting for the parent record to be created
|
31
|
+
entry.import_attempts += 1
|
32
|
+
entry.save!
|
33
|
+
if entry.import_attempts < 5
|
34
|
+
ImportFileSetJob
|
35
|
+
.set(wait: (entry.import_attempts + 1).minutes)
|
36
|
+
.perform_later(entry_id, importer_run_id)
|
37
|
+
else
|
38
|
+
ImporterRun.find(importer_run_id).decrement!(:enqueued_records) # rubocop:disable Rails/SkipsModelValidations
|
39
|
+
entry.status_info(e)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
attr_reader :parent_record
|
46
|
+
|
47
|
+
def validate_parent!(parent_identifier)
|
48
|
+
# if parent_identifier is missing, it will be caught by #validate_presence_of_parent!
|
49
|
+
return if parent_identifier.blank?
|
50
|
+
|
51
|
+
find_parent_record(parent_identifier)
|
52
|
+
check_parent_exists!(parent_identifier)
|
53
|
+
check_parent_is_a_work!(parent_identifier)
|
54
|
+
end
|
55
|
+
|
56
|
+
def check_parent_exists!(parent_identifier)
|
57
|
+
raise MissingParentError, %(Unable to find a record with the identifier "#{parent_identifier}") if parent_record.blank?
|
58
|
+
end
|
59
|
+
|
60
|
+
def check_parent_is_a_work!(parent_identifier)
|
61
|
+
error_msg = %(A record with the ID "#{parent_identifier}" was found, but it was a #{parent_record.class}, which is not an valid/available work type)
|
62
|
+
raise ::StandardError, error_msg unless curation_concern?(parent_record)
|
63
|
+
end
|
64
|
+
|
65
|
+
def find_parent_record(parent_identifier)
|
66
|
+
@parent_record ||= find_record(parent_identifier)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -10,11 +10,13 @@ module Bulkrax
|
|
10
10
|
entry.build
|
11
11
|
if entry.status == "Complete"
|
12
12
|
ImporterRun.find(args[1]).increment!(:processed_records)
|
13
|
+
ImporterRun.find(args[1]).increment!(:processed_works)
|
13
14
|
ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
|
14
15
|
else
|
15
16
|
# do not retry here because whatever parse error kept you from creating a work will likely
|
16
17
|
# keep preventing you from doing so.
|
17
18
|
ImporterRun.find(args[1]).increment!(:failed_records)
|
19
|
+
ImporterRun.find(args[1]).increment!(:failed_works)
|
18
20
|
ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
|
19
21
|
end
|
20
22
|
entry.save!
|