bulkrax 8.1.0 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/javascripts/bulkrax/datatables.js +1 -1
- data/app/controllers/bulkrax/importers_controller.rb +9 -1
- data/app/factories/bulkrax/object_factory.rb +10 -14
- data/app/factories/bulkrax/valkyrie_object_factory.rb +105 -14
- data/app/jobs/bulkrax/create_relationships_job.rb +24 -17
- data/app/jobs/bulkrax/delete_file_set_job.rb +19 -1
- data/app/models/bulkrax/importer.rb +37 -0
- data/app/models/bulkrax/importer_run.rb +11 -0
- data/app/models/bulkrax/pending_relationship.rb +2 -0
- data/app/models/concerns/bulkrax/has_matchers.rb +1 -0
- data/app/parsers/bulkrax/application_parser.rb +1 -1
- data/app/views/bulkrax/importers/_csv_fields.html.erb +3 -1
- data/app/views/bulkrax/importers/show.html.erb +5 -5
- data/config/routes.rb +1 -0
- data/db/migrate/20240806161142_add_file_name_to_uploaded_files.rb +5 -0
- data/db/migrate/20240823173525_add_error_tracking_to_pending_relationships.rb +5 -0
- data/db/migrate/20240916182737_add_last_imported_at_to_bulkrax_importers.rb +5 -0
- data/db/migrate/20240916182823_add_next_import_at_to_bulkrax_importers.rb +5 -0
- data/lib/bulkrax/version.rb +1 -1
- data/lib/tasks/bulkrax_tasks.rake +21 -0
- metadata +6 -3
- data/app/factories/bulkrax/valkyrize-hyku.code-workspace +0 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7886fb5eefc500cc8e567d9097e6930e67cee2908fa7fa0d79ce05fc714db392
|
4
|
+
data.tar.gz: c6e720816dd152931df91e06be37977216a7ef06875a98386350706e4df1e3ea
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 62971f4eab58de239643d016ec8f8701b6e71953bbb8ccc4bf1d74fd8a88d3798c506004704203107e6f676218bc0dcd337776ecccc423b6c932aeddcd0dacf5
|
7
|
+
data.tar.gz: f3710a5e394a470753ea83d5e007505afd91a8fcdc3c383465fdaa7f2eb9c0901b3bef7cb69ae2c9bfa6d508231095e44ac7781cc9c52565162adfe2e58a5c3d
|
@@ -67,7 +67,7 @@ Blacklight.onLoad(function() {
|
|
67
67
|
{ "data": "name" },
|
68
68
|
{ "data": "status_message" },
|
69
69
|
{ "data": "created_at" },
|
70
|
-
{ "data": "download" },
|
70
|
+
{ "data": "download", "orderable": false },
|
71
71
|
{ "data": "actions", "orderable": false }
|
72
72
|
],
|
73
73
|
initComplete: function () {
|
@@ -13,7 +13,7 @@ module Bulkrax
|
|
13
13
|
before_action :token_authenticate!, if: -> { api_request? }, only: [:create, :update, :delete]
|
14
14
|
before_action :authenticate_user!, unless: -> { api_request? }
|
15
15
|
before_action :check_permissions
|
16
|
-
before_action :set_importer, only: [:show, :entry_table, :edit, :update, :destroy]
|
16
|
+
before_action :set_importer, only: [:show, :entry_table, :edit, :update, :destroy, :original_file]
|
17
17
|
with_themed_layout 'dashboard' if defined?(::Hyrax)
|
18
18
|
|
19
19
|
# GET /importers
|
@@ -201,6 +201,14 @@ module Bulkrax
|
|
201
201
|
end
|
202
202
|
end
|
203
203
|
|
204
|
+
def original_file
|
205
|
+
if @importer.original_file?
|
206
|
+
send_file @importer.original_file
|
207
|
+
else
|
208
|
+
redirect_to @importer, alert: 'Importer does not support file re-download or the imported file is not found on the server.'
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
204
212
|
# GET /importers/1/export_errors
|
205
213
|
def export_errors
|
206
214
|
@importer = Importer.find(params[:importer_id])
|
@@ -12,7 +12,7 @@ module Bulkrax
|
|
12
12
|
# @note This does not save either object. We need to do that in another
|
13
13
|
# loop. Why? Because we might be adding many items to the parent.
|
14
14
|
def self.add_child_to_parent_work(parent:, child:)
|
15
|
-
return true if parent.ordered_members.to_a.include?(
|
15
|
+
return true if parent.ordered_members.to_a.include?(child)
|
16
16
|
|
17
17
|
parent.ordered_members << child
|
18
18
|
end
|
@@ -228,33 +228,29 @@ module Bulkrax
|
|
228
228
|
actor.create_metadata(attrs)
|
229
229
|
actor.create_content(uploaded_file) if uploaded_file
|
230
230
|
actor.attach_to_work(work, attrs)
|
231
|
-
handle_remote_file(remote_file: remote_file, actor: actor
|
231
|
+
handle_remote_file(remote_file: remote_file, actor: actor) if remote_file
|
232
232
|
end
|
233
233
|
|
234
234
|
def update_file_set(attrs)
|
235
235
|
file_set_attrs = attrs.slice(*object.attributes.keys)
|
236
236
|
actor = ::Hyrax::Actors::FileSetActor.new(object, @user)
|
237
237
|
attrs['remote_files']&.each do |remote_file|
|
238
|
-
handle_remote_file(remote_file: remote_file, actor: actor
|
238
|
+
handle_remote_file(remote_file: remote_file, actor: actor)
|
239
239
|
end
|
240
240
|
actor.update_metadata(file_set_attrs)
|
241
241
|
end
|
242
242
|
|
243
|
-
def handle_remote_file(remote_file:, actor
|
243
|
+
def handle_remote_file(remote_file:, actor:)
|
244
244
|
actor.file_set.label = remote_file['file_name']
|
245
245
|
actor.file_set.import_url = remote_file['url']
|
246
|
+
auth_header = remote_file.fetch('auth_header', {})
|
246
247
|
|
247
|
-
|
248
|
-
|
249
|
-
tmp_file.binmode
|
250
|
-
|
251
|
-
URI.open(url) do |url_file|
|
252
|
-
tmp_file.write(url_file.read)
|
253
|
-
end
|
248
|
+
ImportUrlJob.perform_now(actor.file_set, file_set_operation_for(user: @user), auth_header)
|
249
|
+
end
|
254
250
|
|
255
|
-
|
256
|
-
|
257
|
-
|
251
|
+
def file_set_operation_for(user:)
|
252
|
+
Hyrax::Operation.create!(user: user,
|
253
|
+
operation_type: "Attach Remote File")
|
258
254
|
end
|
259
255
|
end
|
260
256
|
# rubocop:enable Metrics/ClassLength
|
@@ -217,7 +217,7 @@ module Bulkrax
|
|
217
217
|
run
|
218
218
|
# reload the object
|
219
219
|
object = find
|
220
|
-
return object if object
|
220
|
+
return object if object&.persisted?
|
221
221
|
|
222
222
|
raise(ObjectFactoryInterface::RecordInvalid, object)
|
223
223
|
end
|
@@ -225,10 +225,10 @@ module Bulkrax
|
|
225
225
|
private
|
226
226
|
|
227
227
|
def apply_depositor_metadata
|
228
|
-
return if object.depositor.present?
|
228
|
+
return if @object.depositor.present?
|
229
229
|
|
230
|
-
object.depositor = @user.email
|
231
|
-
object = Hyrax.persister.save(resource: object)
|
230
|
+
@object.depositor = @user.email
|
231
|
+
object = Hyrax.persister.save(resource: @object)
|
232
232
|
self.class.publish(event: "object.metadata.updated", object: object, user: @user)
|
233
233
|
object
|
234
234
|
end
|
@@ -249,19 +249,77 @@ module Bulkrax
|
|
249
249
|
end
|
250
250
|
|
251
251
|
def create_work(attrs)
|
252
|
-
# NOTE: We do not add relationships here; that is part of the create
|
253
|
-
|
252
|
+
# NOTE: We do not add relationships here; that is part of the create relationships job.
|
253
|
+
attrs = HashWithIndifferentAccess.new(attrs)
|
254
254
|
perform_transaction_for(object: object, attrs: attrs) do
|
255
|
+
uploaded_files, file_set_params = prep_fileset_content(attrs)
|
255
256
|
transactions["change_set.create_work"]
|
256
257
|
.with_step_args(
|
257
|
-
'work_resource.add_file_sets' => { uploaded_files:
|
258
|
+
'work_resource.add_file_sets' => { uploaded_files: uploaded_files, file_set_params: file_set_params },
|
258
259
|
"change_set.set_user_as_depositor" => { user: @user },
|
259
260
|
"work_resource.change_depositor" => { user: @user },
|
260
|
-
'work_resource.save_acl' => { permissions_params: [attrs
|
261
|
+
'work_resource.save_acl' => { permissions_params: [attrs.try('visibility') || 'open'].compact }
|
261
262
|
)
|
262
263
|
end
|
263
264
|
end
|
264
265
|
|
266
|
+
## Prepare fileset data in the required format for creating or updating a work
|
267
|
+
# TODO: Determine why attrs is different from attributes?
|
268
|
+
# TODO: Disabled s3 until we get additional details
|
269
|
+
def prep_fileset_content(attrs)
|
270
|
+
# combine remote_files + thumbnail_url [Array < { url:, file_name:, * }]
|
271
|
+
thumbnail_url = HashWithIndifferentAccess.new(self.attributes)['thumbnail_url']
|
272
|
+
all_remote_files = merge_thumbnails(remote_files: attrs["remote_files"], thumbnail_url: thumbnail_url)
|
273
|
+
# combine local & remote files [Array < Hash &/or String]
|
274
|
+
all_local_files = self.attributes['file'] || []
|
275
|
+
all_files = all_local_files + all_remote_files
|
276
|
+
|
277
|
+
# collect all uploaded files [Array < Hyrax::UploadedFile]
|
278
|
+
uploaded_local = uploaded_local_files(uploaded_files: attrs[:uploaded_files])
|
279
|
+
uploaded_remote = uploaded_remote_files(remote_files: all_remote_files)
|
280
|
+
# uploaded_s3 = uploaded_s3_files(remote_files: attrs[:remote_files])
|
281
|
+
uploaded_files = uploaded_local + uploaded_remote
|
282
|
+
|
283
|
+
# add in other attributes
|
284
|
+
file_set_params = file_set_params_for(uploads: uploaded_files, files: all_files)
|
285
|
+
# return data for filesets
|
286
|
+
[uploaded_files, file_set_params]
|
287
|
+
end
|
288
|
+
|
289
|
+
# supports using thumbnail_url to import a thumbnail separately from other remote_files
|
290
|
+
# in the format thumbnail_url: { url:, file_name: }
|
291
|
+
def merge_thumbnails(remote_files:, thumbnail_url:)
|
292
|
+
r = remote_files || []
|
293
|
+
thumbnail_url.present? ? r + [thumbnail_url] : r
|
294
|
+
end
|
295
|
+
|
296
|
+
# formats file info and facilitates additional custom file_set attributes
|
297
|
+
# To have the additional attributes appear on the file_set, they must be:
|
298
|
+
# - included in the file_set_metadata.yaml
|
299
|
+
# - overridden in file_set_args from Hyrax::WorkUploadsHandler
|
300
|
+
# @param uploads [Array < Hyrax::UploadedFile]
|
301
|
+
# @param files [Array < Hash or String]
|
302
|
+
# @return [Array < Hash]
|
303
|
+
def file_set_params_for(uploads:, files:)
|
304
|
+
# remove url, file_name and paths from attributes
|
305
|
+
additional_attributes = files.map do |f|
|
306
|
+
case f
|
307
|
+
when String
|
308
|
+
{}
|
309
|
+
else
|
310
|
+
temp = f.reject { |key, _| key.to_s == 'url' || key.to_s == 'file_name' }
|
311
|
+
temp['import_url'] = f['url']
|
312
|
+
temp
|
313
|
+
end
|
314
|
+
end
|
315
|
+
|
316
|
+
file_attrs = []
|
317
|
+
uploads.each_with_index do |f, index|
|
318
|
+
file_attrs << ({ uploaded_file_id: f["id"].to_s, filename: files[index]["file_name"] }).merge(additional_attributes[index])
|
319
|
+
end
|
320
|
+
file_attrs.compact.uniq
|
321
|
+
end
|
322
|
+
|
265
323
|
def create_collection(attrs)
|
266
324
|
# TODO: Handle Collection Type
|
267
325
|
#
|
@@ -328,10 +386,12 @@ module Bulkrax
|
|
328
386
|
end
|
329
387
|
|
330
388
|
def update_work(attrs)
|
389
|
+
attrs = HashWithIndifferentAccess.new(attrs)
|
331
390
|
perform_transaction_for(object: object, attrs: attrs) do
|
391
|
+
uploaded_files, file_set_params = prep_fileset_content(attrs)
|
332
392
|
transactions["change_set.update_work"]
|
333
393
|
.with_step_args(
|
334
|
-
'work_resource.add_file_sets' => { uploaded_files:
|
394
|
+
'work_resource.add_file_sets' => { uploaded_files: uploaded_files, file_set_params: file_set_params },
|
335
395
|
'work_resource.save_acl' => { permissions_params: [attrs.try('visibility') || 'open'].compact }
|
336
396
|
)
|
337
397
|
end
|
@@ -349,17 +409,13 @@ module Bulkrax
|
|
349
409
|
# TODO: Make it work
|
350
410
|
end
|
351
411
|
|
352
|
-
def uploaded_files_from(attrs)
|
353
|
-
uploaded_local_files(uploaded_files: attrs[:uploaded_files]) + uploaded_s3_files(remote_files: attrs[:remote_files])
|
354
|
-
end
|
355
|
-
|
356
412
|
def uploaded_local_files(uploaded_files: [])
|
357
413
|
Array.wrap(uploaded_files).map do |file_id|
|
358
414
|
Hyrax::UploadedFile.find(file_id)
|
359
415
|
end
|
360
416
|
end
|
361
417
|
|
362
|
-
def uploaded_s3_files(remote_files:
|
418
|
+
def uploaded_s3_files(remote_files: [])
|
363
419
|
return [] if remote_files.blank?
|
364
420
|
|
365
421
|
s3_bucket_name = ENV.fetch("STAGING_AREA_S3_BUCKET", "comet-staging-area-#{Rails.env}")
|
@@ -371,6 +427,41 @@ module Bulkrax
|
|
371
427
|
end.compact
|
372
428
|
end
|
373
429
|
|
430
|
+
def uploaded_remote_files(remote_files: [])
|
431
|
+
remote_files.map do |r|
|
432
|
+
file_path = download_file(r["url"])
|
433
|
+
next unless file_path
|
434
|
+
|
435
|
+
create_uploaded_file(file_path, r["file_name"])
|
436
|
+
end.compact
|
437
|
+
end
|
438
|
+
|
439
|
+
def download_file(url)
|
440
|
+
require 'open-uri'
|
441
|
+
require 'tempfile'
|
442
|
+
|
443
|
+
begin
|
444
|
+
file = Tempfile.new
|
445
|
+
file.binmode
|
446
|
+
file.write(URI.open(url).read)
|
447
|
+
file.rewind
|
448
|
+
file.path
|
449
|
+
rescue => e
|
450
|
+
Rails.logger.debug "Failed to download file from #{url}: #{e.message}"
|
451
|
+
nil
|
452
|
+
end
|
453
|
+
end
|
454
|
+
|
455
|
+
def create_uploaded_file(file_path, file_name)
|
456
|
+
file = File.open(file_path)
|
457
|
+
uploaded_file = Hyrax::UploadedFile.create(file: file, user: @user, filename: file_name)
|
458
|
+
file.close
|
459
|
+
uploaded_file
|
460
|
+
rescue => e
|
461
|
+
Rails.logger.debug "Failed to create Hyrax::UploadedFile for #{file_name}: #{e.message}"
|
462
|
+
nil
|
463
|
+
end
|
464
|
+
|
374
465
|
# @Override Destroy existing files with Hyrax::Transactions
|
375
466
|
def destroy_existing_files
|
376
467
|
existing_files = Hyrax.custom_queries.find_child_file_sets(resource: object)
|
@@ -38,11 +38,13 @@ module Bulkrax
|
|
38
38
|
#
|
39
39
|
# @see https://github.com/scientist-softserv/louisville-hyku/commit/128a9ef
|
40
40
|
class_attribute :update_child_records_works_file_sets, default: false
|
41
|
+
class_attribute :max_failure_count, default: 5
|
41
42
|
|
42
43
|
include DynamicRecordLookup
|
43
44
|
|
44
45
|
queue_as Bulkrax.config.ingest_queue_name
|
45
46
|
|
47
|
+
attr_accessor :user, :importer_run, :errors
|
46
48
|
##
|
47
49
|
# @param parent_identifier [String] Work/Collection ID or Bulkrax::Entry source_identifiers
|
48
50
|
# @param importer_run [Bulkrax::ImporterRun] current importer run (needed to properly update counters)
|
@@ -54,9 +56,10 @@ module Bulkrax
|
|
54
56
|
# is the child in the relationship, and vice versa if a child_identifier is passed.
|
55
57
|
#
|
56
58
|
# rubocop:disable Metrics/MethodLength
|
57
|
-
def perform(parent_identifier:, importer_run_id:) # rubocop:disable Metrics/AbcSize
|
58
|
-
|
59
|
-
|
59
|
+
def perform(parent_identifier:, importer_run_id: nil, run_user: nil, failure_count: 0) # rubocop:disable Metrics/AbcSize
|
60
|
+
importer_run = Bulkrax::ImporterRun.find(importer_run_id) if importer_run_id
|
61
|
+
user = run_user || importer_run&.user
|
62
|
+
ability = Ability.new(user)
|
60
63
|
|
61
64
|
parent_entry, parent_record = find_record(parent_identifier, importer_run_id)
|
62
65
|
|
@@ -69,19 +72,21 @@ module Bulkrax
|
|
69
72
|
if parent_record
|
70
73
|
conditionally_acquire_lock_for(parent_record.id) do
|
71
74
|
ActiveRecord::Base.uncached do
|
72
|
-
Bulkrax::PendingRelationship.where(parent_id: parent_identifier
|
75
|
+
Bulkrax::PendingRelationship.where(parent_id: parent_identifier)
|
73
76
|
.ordered.find_each do |rel|
|
74
77
|
process(relationship: rel, importer_run_id: importer_run_id, parent_record: parent_record, ability: ability)
|
75
78
|
number_of_successes += 1
|
79
|
+
@parent_record_members_added = true
|
76
80
|
rescue => e
|
77
81
|
number_of_failures += 1
|
82
|
+
rel.set_status_info(e, importer_run)
|
78
83
|
errors << e
|
79
84
|
end
|
80
85
|
end
|
81
86
|
|
82
87
|
# save record if members were added
|
83
88
|
if @parent_record_members_added
|
84
|
-
Bulkrax.object_factory.save!(resource: parent_record, user:
|
89
|
+
Bulkrax.object_factory.save!(resource: parent_record, user: user)
|
85
90
|
Bulkrax.object_factory.publish(event: 'object.membership.updated', object: parent_record)
|
86
91
|
Bulkrax.object_factory.update_index(resources: @child_members_added)
|
87
92
|
end
|
@@ -104,10 +109,17 @@ module Bulkrax
|
|
104
109
|
# rubocop:enable Rails/SkipsModelValidations
|
105
110
|
|
106
111
|
parent_entry&.set_status_info(errors.last, importer_run)
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
112
|
+
failure_count += 1
|
113
|
+
|
114
|
+
if failure_count < max_failure_count
|
115
|
+
reschedule(
|
116
|
+
parent_identifier: parent_identifier,
|
117
|
+
importer_run_id: importer_run_id,
|
118
|
+
run_user: run_user,
|
119
|
+
failure_count: failure_count
|
120
|
+
)
|
121
|
+
end
|
122
|
+
return errors # stop current job from continuing to run after rescheduling
|
111
123
|
else
|
112
124
|
# rubocop:disable Rails/SkipsModelValidations
|
113
125
|
ImporterRun.update_counters(importer_run_id, processed_relationships: number_of_successes)
|
@@ -116,8 +128,6 @@ module Bulkrax
|
|
116
128
|
end
|
117
129
|
# rubocop:enable Metrics/MethodLength
|
118
130
|
|
119
|
-
attr_reader :importer_run
|
120
|
-
|
121
131
|
private
|
122
132
|
|
123
133
|
##
|
@@ -170,7 +180,7 @@ module Bulkrax
|
|
170
180
|
Bulkrax.object_factory.add_resource_to_collection(
|
171
181
|
collection: parent_record,
|
172
182
|
resource: child_record,
|
173
|
-
user:
|
183
|
+
user: user
|
174
184
|
)
|
175
185
|
end
|
176
186
|
|
@@ -183,11 +193,8 @@ module Bulkrax
|
|
183
193
|
)
|
184
194
|
end
|
185
195
|
|
186
|
-
def reschedule(
|
187
|
-
CreateRelationshipsJob.set(wait: 10.minutes).perform_later(
|
188
|
-
parent_identifier: parent_identifier,
|
189
|
-
importer_run_id: importer_run_id
|
190
|
-
)
|
196
|
+
def reschedule(**kargs)
|
197
|
+
CreateRelationshipsJob.set(wait: 10.minutes).perform_later(**kargs)
|
191
198
|
end
|
192
199
|
end
|
193
200
|
end
|
@@ -1,5 +1,23 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module Bulkrax
|
4
|
-
class DeleteFileSetJob < DeleteJob
|
4
|
+
class DeleteFileSetJob < DeleteJob
|
5
|
+
def perform(entry, importer_run)
|
6
|
+
file_set = entry.factory.find
|
7
|
+
if file_set
|
8
|
+
parent = file_set.parent
|
9
|
+
if parent&.respond_to?(:ordered_members)
|
10
|
+
om = parent.ordered_members.to_a
|
11
|
+
om.delete(file_set)
|
12
|
+
parent.ordered_members = om
|
13
|
+
elsif parent.respond_to?(:member_ids)
|
14
|
+
parent.member_ids.delete(file_set.id)
|
15
|
+
Hyrax.persister.save(resource: parent)
|
16
|
+
end
|
17
|
+
parent.save
|
18
|
+
end
|
19
|
+
|
20
|
+
super
|
21
|
+
end
|
22
|
+
end
|
5
23
|
end
|
@@ -18,6 +18,9 @@ module Bulkrax
|
|
18
18
|
|
19
19
|
delegate :create_parent_child_relationships, :valid_import?, :write_errored_entries_file, :visibility, to: :parser
|
20
20
|
|
21
|
+
after_save :set_last_imported_at_from_importer_run
|
22
|
+
after_save :set_next_import_at_from_importer_run
|
23
|
+
|
21
24
|
attr_accessor :only_updates, :file_style, :file
|
22
25
|
attr_writer :current_run
|
23
26
|
|
@@ -149,6 +152,18 @@ module Bulkrax
|
|
149
152
|
@seen ||= {}
|
150
153
|
end
|
151
154
|
|
155
|
+
def import_file_path
|
156
|
+
self.parser_fields['import_file_path']
|
157
|
+
end
|
158
|
+
|
159
|
+
def original_file?
|
160
|
+
import_file_path && File.exist?(import_file_path)
|
161
|
+
end
|
162
|
+
|
163
|
+
def original_file
|
164
|
+
import_file_path if original_file?
|
165
|
+
end
|
166
|
+
|
152
167
|
def replace_files
|
153
168
|
self.parser_fields['replace_files']
|
154
169
|
end
|
@@ -235,5 +250,27 @@ module Bulkrax
|
|
235
250
|
rescue
|
236
251
|
"#{self.id}_#{self.created_at.strftime('%Y%m%d%H%M%S')}"
|
237
252
|
end
|
253
|
+
|
254
|
+
private
|
255
|
+
|
256
|
+
# Adding this here since we can update the importer without running the importer.
|
257
|
+
# When we simply save the importer (as in just updating the importer from the options),
|
258
|
+
# it does not trigger the after_save callback in the importer_run.
|
259
|
+
def set_last_imported_at_from_importer_run
|
260
|
+
return if @skip_set_last_imported_at # Prevent infinite loop
|
261
|
+
|
262
|
+
@skip_set_last_imported_at = true
|
263
|
+
importer_runs.last&.set_last_imported_at
|
264
|
+
@skip_set_last_imported_at = false
|
265
|
+
end
|
266
|
+
|
267
|
+
# @see #set_last_imported_at_from_importer_run
|
268
|
+
def set_next_import_at_from_importer_run
|
269
|
+
return if @skip_set_next_import_at # Prevent infinite loop
|
270
|
+
|
271
|
+
@skip_set_next_import_at = true
|
272
|
+
importer_runs.last&.set_next_import_at
|
273
|
+
@skip_set_next_import_at = false
|
274
|
+
end
|
238
275
|
end
|
239
276
|
end
|
@@ -6,6 +6,9 @@ module Bulkrax
|
|
6
6
|
has_many :statuses, as: :runnable, dependent: :destroy
|
7
7
|
has_many :pending_relationships, dependent: :destroy
|
8
8
|
|
9
|
+
after_save :set_last_imported_at
|
10
|
+
after_save :set_next_import_at
|
11
|
+
|
9
12
|
def parents
|
10
13
|
pending_relationships.pluck(:parent_id).uniq
|
11
14
|
end
|
@@ -15,5 +18,13 @@ module Bulkrax
|
|
15
18
|
# fallback to the configured user.
|
16
19
|
importer.user || Bulkrax.fallback_user_for_importer_exporter_processing
|
17
20
|
end
|
21
|
+
|
22
|
+
def set_last_imported_at
|
23
|
+
importer.update(last_imported_at: importer.last_imported_at)
|
24
|
+
end
|
25
|
+
|
26
|
+
def set_next_import_at
|
27
|
+
importer.update(next_import_at: importer.next_import_at)
|
28
|
+
end
|
18
29
|
end
|
19
30
|
end
|
@@ -150,7 +150,7 @@ module Bulkrax
|
|
150
150
|
end
|
151
151
|
end
|
152
152
|
|
153
|
-
# The visibility of the record. Acceptable values are: "open", "
|
153
|
+
# The visibility of the record. Acceptable values are: "open", "embargo", "lease", "authenticated", "restricted". The default is "open"
|
154
154
|
#
|
155
155
|
# @return [String]
|
156
156
|
# @see https://github.com/samvera/hydra-head/blob/main/hydra-access-controls/app/models/concerns/hydra/access_controls/access_right.rb Hydra::AccessControls::AccessRight for details on the range of values.
|
@@ -1,13 +1,15 @@
|
|
1
1
|
<div class='csv_fields'>
|
2
2
|
|
3
3
|
<%= fi.input :visibility,
|
4
|
+
label: 'Default Visibility',
|
4
5
|
collection: [
|
5
6
|
['Public', 'open'],
|
6
7
|
['Private', 'restricted'],
|
7
8
|
['Institution', 'authenticated']
|
8
9
|
],
|
9
10
|
selected: importer.parser_fields['visibility'] || 'open',
|
10
|
-
input_html: { class: 'form-control' }
|
11
|
+
input_html: { class: 'form-control' },
|
12
|
+
hint: 'If your CSV includes the visibility field, it will override the default setting.'
|
11
13
|
%>
|
12
14
|
|
13
15
|
<% if defined?(::Hyrax) %>
|
@@ -1,12 +1,12 @@
|
|
1
1
|
<div class="col-xs-12 main-header">
|
2
2
|
<h1><span class="fa fa-cloud-upload" aria-hidden="true"></span> Importer: <%= @importer.name %></h1>
|
3
|
-
|
4
|
-
|
5
|
-
|
3
|
+
<div class="pull-right">
|
4
|
+
<%= link_to 'Download Original File', importer_original_file_path(@importer.id), class: 'btn btn-primary', data: { turbolinks: false } if @importer.original_file %>
|
5
|
+
<% if @importer.failed_entries? %>
|
6
6
|
<%= link_to 'Export Errored Entries', importer_export_errors_path(@importer.id), class: 'btn btn-primary', data: { turbolinks: false }%>
|
7
7
|
<%= link_to 'Upload Corrected Entries', importer_upload_corrected_entries_path(@importer.id), class: 'btn btn-primary' if @importer.parser.is_a?(Bulkrax::CsvParser) %>
|
8
|
-
|
9
|
-
|
8
|
+
<% end %>
|
9
|
+
</div>
|
10
10
|
</div>
|
11
11
|
<div class="panel panel-default bulkrax-align-text">
|
12
12
|
<div class="panel-body">
|
data/config/routes.rb
CHANGED
data/lib/bulkrax/version.rb
CHANGED
@@ -141,4 +141,25 @@ namespace :bulkrax do
|
|
141
141
|
rescue => e
|
142
142
|
puts "(#{e.message})"
|
143
143
|
end
|
144
|
+
|
145
|
+
desc "Resave importers"
|
146
|
+
task resave_importers: :environment do
|
147
|
+
if defined?(::Hyku)
|
148
|
+
Account.find_each do |account|
|
149
|
+
next if account.name == "search"
|
150
|
+
switch!(account)
|
151
|
+
puts "=============== updating #{account.name} ============"
|
152
|
+
|
153
|
+
resave_importers
|
154
|
+
|
155
|
+
puts "=============== finished updating #{account.name} ============"
|
156
|
+
end
|
157
|
+
else
|
158
|
+
resave_importers
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
def resave_importers
|
163
|
+
Bulkrax::Importer.find_each(&:save!)
|
164
|
+
end
|
144
165
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bulkrax
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 8.
|
4
|
+
version: 8.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rob Kaufman
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-09-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|
@@ -345,7 +345,6 @@ files:
|
|
345
345
|
- app/factories/bulkrax/object_factory.rb
|
346
346
|
- app/factories/bulkrax/object_factory_interface.rb
|
347
347
|
- app/factories/bulkrax/valkyrie_object_factory.rb
|
348
|
-
- app/factories/bulkrax/valkyrize-hyku.code-workspace
|
349
348
|
- app/helpers/bulkrax/application_helper.rb
|
350
349
|
- app/helpers/bulkrax/exporters_helper.rb
|
351
350
|
- app/helpers/bulkrax/importers_helper.rb
|
@@ -483,6 +482,10 @@ files:
|
|
483
482
|
- db/migrate/20240208005801_denormalize_status_message.rb
|
484
483
|
- db/migrate/20240209070952_update_identifier_index.rb
|
485
484
|
- db/migrate/20240307053156_add_index_to_metadata_bulkrax_identifier.rb
|
485
|
+
- db/migrate/20240806161142_add_file_name_to_uploaded_files.rb
|
486
|
+
- db/migrate/20240823173525_add_error_tracking_to_pending_relationships.rb
|
487
|
+
- db/migrate/20240916182737_add_last_imported_at_to_bulkrax_importers.rb
|
488
|
+
- db/migrate/20240916182823_add_next_import_at_to_bulkrax_importers.rb
|
486
489
|
- lib/bulkrax.rb
|
487
490
|
- lib/bulkrax/engine.rb
|
488
491
|
- lib/bulkrax/entry_spec_helper.rb
|
@@ -1,19 +0,0 @@
|
|
1
|
-
{
|
2
|
-
"folders": [
|
3
|
-
{
|
4
|
-
"path": "../../../../../hyku"
|
5
|
-
},
|
6
|
-
{
|
7
|
-
"path": "../../../../hyrax"
|
8
|
-
},
|
9
|
-
{
|
10
|
-
"path": "../../.."
|
11
|
-
},
|
12
|
-
{
|
13
|
-
"path": "../../../../iiif_print"
|
14
|
-
}
|
15
|
-
],
|
16
|
-
"settings": {
|
17
|
-
"github.copilot.inlineSuggest.enable": true
|
18
|
-
}
|
19
|
-
}
|