bulkrax 8.1.0 → 8.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/assets/javascripts/bulkrax/datatables.js +1 -1
- data/app/controllers/bulkrax/importers_controller.rb +9 -1
- data/app/factories/bulkrax/object_factory.rb +10 -14
- data/app/factories/bulkrax/valkyrie_object_factory.rb +105 -14
- data/app/jobs/bulkrax/create_relationships_job.rb +24 -17
- data/app/jobs/bulkrax/delete_file_set_job.rb +19 -1
- data/app/models/bulkrax/importer.rb +37 -0
- data/app/models/bulkrax/importer_run.rb +11 -0
- data/app/models/bulkrax/pending_relationship.rb +2 -0
- data/app/models/concerns/bulkrax/has_matchers.rb +1 -0
- data/app/parsers/bulkrax/application_parser.rb +1 -1
- data/app/views/bulkrax/importers/_csv_fields.html.erb +3 -1
- data/app/views/bulkrax/importers/show.html.erb +5 -5
- data/config/routes.rb +1 -0
- data/db/migrate/20240806161142_add_file_name_to_uploaded_files.rb +5 -0
- data/db/migrate/20240823173525_add_error_tracking_to_pending_relationships.rb +5 -0
- data/db/migrate/20240916182737_add_last_imported_at_to_bulkrax_importers.rb +5 -0
- data/db/migrate/20240916182823_add_next_import_at_to_bulkrax_importers.rb +5 -0
- data/lib/bulkrax/version.rb +1 -1
- data/lib/tasks/bulkrax_tasks.rake +21 -0
- metadata +6 -3
- data/app/factories/bulkrax/valkyrize-hyku.code-workspace +0 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7886fb5eefc500cc8e567d9097e6930e67cee2908fa7fa0d79ce05fc714db392
|
4
|
+
data.tar.gz: c6e720816dd152931df91e06be37977216a7ef06875a98386350706e4df1e3ea
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 62971f4eab58de239643d016ec8f8701b6e71953bbb8ccc4bf1d74fd8a88d3798c506004704203107e6f676218bc0dcd337776ecccc423b6c932aeddcd0dacf5
|
7
|
+
data.tar.gz: f3710a5e394a470753ea83d5e007505afd91a8fcdc3c383465fdaa7f2eb9c0901b3bef7cb69ae2c9bfa6d508231095e44ac7781cc9c52565162adfe2e58a5c3d
|
@@ -67,7 +67,7 @@ Blacklight.onLoad(function() {
|
|
67
67
|
{ "data": "name" },
|
68
68
|
{ "data": "status_message" },
|
69
69
|
{ "data": "created_at" },
|
70
|
-
{ "data": "download" },
|
70
|
+
{ "data": "download", "orderable": false },
|
71
71
|
{ "data": "actions", "orderable": false }
|
72
72
|
],
|
73
73
|
initComplete: function () {
|
@@ -13,7 +13,7 @@ module Bulkrax
|
|
13
13
|
before_action :token_authenticate!, if: -> { api_request? }, only: [:create, :update, :delete]
|
14
14
|
before_action :authenticate_user!, unless: -> { api_request? }
|
15
15
|
before_action :check_permissions
|
16
|
-
before_action :set_importer, only: [:show, :entry_table, :edit, :update, :destroy]
|
16
|
+
before_action :set_importer, only: [:show, :entry_table, :edit, :update, :destroy, :original_file]
|
17
17
|
with_themed_layout 'dashboard' if defined?(::Hyrax)
|
18
18
|
|
19
19
|
# GET /importers
|
@@ -201,6 +201,14 @@ module Bulkrax
|
|
201
201
|
end
|
202
202
|
end
|
203
203
|
|
204
|
+
def original_file
|
205
|
+
if @importer.original_file?
|
206
|
+
send_file @importer.original_file
|
207
|
+
else
|
208
|
+
redirect_to @importer, alert: 'Importer does not support file re-download or the imported file is not found on the server.'
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
204
212
|
# GET /importers/1/export_errors
|
205
213
|
def export_errors
|
206
214
|
@importer = Importer.find(params[:importer_id])
|
@@ -12,7 +12,7 @@ module Bulkrax
|
|
12
12
|
# @note This does not save either object. We need to do that in another
|
13
13
|
# loop. Why? Because we might be adding many items to the parent.
|
14
14
|
def self.add_child_to_parent_work(parent:, child:)
|
15
|
-
return true if parent.ordered_members.to_a.include?(
|
15
|
+
return true if parent.ordered_members.to_a.include?(child)
|
16
16
|
|
17
17
|
parent.ordered_members << child
|
18
18
|
end
|
@@ -228,33 +228,29 @@ module Bulkrax
|
|
228
228
|
actor.create_metadata(attrs)
|
229
229
|
actor.create_content(uploaded_file) if uploaded_file
|
230
230
|
actor.attach_to_work(work, attrs)
|
231
|
-
handle_remote_file(remote_file: remote_file, actor: actor
|
231
|
+
handle_remote_file(remote_file: remote_file, actor: actor) if remote_file
|
232
232
|
end
|
233
233
|
|
234
234
|
def update_file_set(attrs)
|
235
235
|
file_set_attrs = attrs.slice(*object.attributes.keys)
|
236
236
|
actor = ::Hyrax::Actors::FileSetActor.new(object, @user)
|
237
237
|
attrs['remote_files']&.each do |remote_file|
|
238
|
-
handle_remote_file(remote_file: remote_file, actor: actor
|
238
|
+
handle_remote_file(remote_file: remote_file, actor: actor)
|
239
239
|
end
|
240
240
|
actor.update_metadata(file_set_attrs)
|
241
241
|
end
|
242
242
|
|
243
|
-
def handle_remote_file(remote_file:, actor
|
243
|
+
def handle_remote_file(remote_file:, actor:)
|
244
244
|
actor.file_set.label = remote_file['file_name']
|
245
245
|
actor.file_set.import_url = remote_file['url']
|
246
|
+
auth_header = remote_file.fetch('auth_header', {})
|
246
247
|
|
247
|
-
|
248
|
-
|
249
|
-
tmp_file.binmode
|
250
|
-
|
251
|
-
URI.open(url) do |url_file|
|
252
|
-
tmp_file.write(url_file.read)
|
253
|
-
end
|
248
|
+
ImportUrlJob.perform_now(actor.file_set, file_set_operation_for(user: @user), auth_header)
|
249
|
+
end
|
254
250
|
|
255
|
-
|
256
|
-
|
257
|
-
|
251
|
+
def file_set_operation_for(user:)
|
252
|
+
Hyrax::Operation.create!(user: user,
|
253
|
+
operation_type: "Attach Remote File")
|
258
254
|
end
|
259
255
|
end
|
260
256
|
# rubocop:enable Metrics/ClassLength
|
@@ -217,7 +217,7 @@ module Bulkrax
|
|
217
217
|
run
|
218
218
|
# reload the object
|
219
219
|
object = find
|
220
|
-
return object if object
|
220
|
+
return object if object&.persisted?
|
221
221
|
|
222
222
|
raise(ObjectFactoryInterface::RecordInvalid, object)
|
223
223
|
end
|
@@ -225,10 +225,10 @@ module Bulkrax
|
|
225
225
|
private
|
226
226
|
|
227
227
|
def apply_depositor_metadata
|
228
|
-
return if object.depositor.present?
|
228
|
+
return if @object.depositor.present?
|
229
229
|
|
230
|
-
object.depositor = @user.email
|
231
|
-
object = Hyrax.persister.save(resource: object)
|
230
|
+
@object.depositor = @user.email
|
231
|
+
object = Hyrax.persister.save(resource: @object)
|
232
232
|
self.class.publish(event: "object.metadata.updated", object: object, user: @user)
|
233
233
|
object
|
234
234
|
end
|
@@ -249,19 +249,77 @@ module Bulkrax
|
|
249
249
|
end
|
250
250
|
|
251
251
|
def create_work(attrs)
|
252
|
-
# NOTE: We do not add relationships here; that is part of the create
|
253
|
-
|
252
|
+
# NOTE: We do not add relationships here; that is part of the create relationships job.
|
253
|
+
attrs = HashWithIndifferentAccess.new(attrs)
|
254
254
|
perform_transaction_for(object: object, attrs: attrs) do
|
255
|
+
uploaded_files, file_set_params = prep_fileset_content(attrs)
|
255
256
|
transactions["change_set.create_work"]
|
256
257
|
.with_step_args(
|
257
|
-
'work_resource.add_file_sets' => { uploaded_files:
|
258
|
+
'work_resource.add_file_sets' => { uploaded_files: uploaded_files, file_set_params: file_set_params },
|
258
259
|
"change_set.set_user_as_depositor" => { user: @user },
|
259
260
|
"work_resource.change_depositor" => { user: @user },
|
260
|
-
'work_resource.save_acl' => { permissions_params: [attrs
|
261
|
+
'work_resource.save_acl' => { permissions_params: [attrs.try('visibility') || 'open'].compact }
|
261
262
|
)
|
262
263
|
end
|
263
264
|
end
|
264
265
|
|
266
|
+
## Prepare fileset data in the required format for creating or updating a work
|
267
|
+
# TODO: Determine why attrs is different from attributes?
|
268
|
+
# TODO: Disabled s3 until we get additional details
|
269
|
+
def prep_fileset_content(attrs)
|
270
|
+
# combine remote_files + thumbnail_url [Array < { url:, file_name:, * }]
|
271
|
+
thumbnail_url = HashWithIndifferentAccess.new(self.attributes)['thumbnail_url']
|
272
|
+
all_remote_files = merge_thumbnails(remote_files: attrs["remote_files"], thumbnail_url: thumbnail_url)
|
273
|
+
# combine local & remote files [Array < Hash &/or String]
|
274
|
+
all_local_files = self.attributes['file'] || []
|
275
|
+
all_files = all_local_files + all_remote_files
|
276
|
+
|
277
|
+
# collect all uploaded files [Array < Hyrax::UploadedFile]
|
278
|
+
uploaded_local = uploaded_local_files(uploaded_files: attrs[:uploaded_files])
|
279
|
+
uploaded_remote = uploaded_remote_files(remote_files: all_remote_files)
|
280
|
+
# uploaded_s3 = uploaded_s3_files(remote_files: attrs[:remote_files])
|
281
|
+
uploaded_files = uploaded_local + uploaded_remote
|
282
|
+
|
283
|
+
# add in other attributes
|
284
|
+
file_set_params = file_set_params_for(uploads: uploaded_files, files: all_files)
|
285
|
+
# return data for filesets
|
286
|
+
[uploaded_files, file_set_params]
|
287
|
+
end
|
288
|
+
|
289
|
+
# supports using thumbnail_url to import a thumbnail separately from other remote_files
|
290
|
+
# in the format thumbnail_url: { url:, file_name: }
|
291
|
+
def merge_thumbnails(remote_files:, thumbnail_url:)
|
292
|
+
r = remote_files || []
|
293
|
+
thumbnail_url.present? ? r + [thumbnail_url] : r
|
294
|
+
end
|
295
|
+
|
296
|
+
# formats file info and facilitates additional custom file_set attributes
|
297
|
+
# To have the additional attributes appear on the file_set, they must be:
|
298
|
+
# - included in the file_set_metadata.yaml
|
299
|
+
# - overridden in file_set_args from Hyrax::WorkUploadsHandler
|
300
|
+
# @param uploads [Array < Hyrax::UploadedFile]
|
301
|
+
# @param files [Array < Hash or String]
|
302
|
+
# @return [Array < Hash]
|
303
|
+
def file_set_params_for(uploads:, files:)
|
304
|
+
# remove url, file_name and paths from attributes
|
305
|
+
additional_attributes = files.map do |f|
|
306
|
+
case f
|
307
|
+
when String
|
308
|
+
{}
|
309
|
+
else
|
310
|
+
temp = f.reject { |key, _| key.to_s == 'url' || key.to_s == 'file_name' }
|
311
|
+
temp['import_url'] = f['url']
|
312
|
+
temp
|
313
|
+
end
|
314
|
+
end
|
315
|
+
|
316
|
+
file_attrs = []
|
317
|
+
uploads.each_with_index do |f, index|
|
318
|
+
file_attrs << ({ uploaded_file_id: f["id"].to_s, filename: files[index]["file_name"] }).merge(additional_attributes[index])
|
319
|
+
end
|
320
|
+
file_attrs.compact.uniq
|
321
|
+
end
|
322
|
+
|
265
323
|
def create_collection(attrs)
|
266
324
|
# TODO: Handle Collection Type
|
267
325
|
#
|
@@ -328,10 +386,12 @@ module Bulkrax
|
|
328
386
|
end
|
329
387
|
|
330
388
|
def update_work(attrs)
|
389
|
+
attrs = HashWithIndifferentAccess.new(attrs)
|
331
390
|
perform_transaction_for(object: object, attrs: attrs) do
|
391
|
+
uploaded_files, file_set_params = prep_fileset_content(attrs)
|
332
392
|
transactions["change_set.update_work"]
|
333
393
|
.with_step_args(
|
334
|
-
'work_resource.add_file_sets' => { uploaded_files:
|
394
|
+
'work_resource.add_file_sets' => { uploaded_files: uploaded_files, file_set_params: file_set_params },
|
335
395
|
'work_resource.save_acl' => { permissions_params: [attrs.try('visibility') || 'open'].compact }
|
336
396
|
)
|
337
397
|
end
|
@@ -349,17 +409,13 @@ module Bulkrax
|
|
349
409
|
# TODO: Make it work
|
350
410
|
end
|
351
411
|
|
352
|
-
def uploaded_files_from(attrs)
|
353
|
-
uploaded_local_files(uploaded_files: attrs[:uploaded_files]) + uploaded_s3_files(remote_files: attrs[:remote_files])
|
354
|
-
end
|
355
|
-
|
356
412
|
def uploaded_local_files(uploaded_files: [])
|
357
413
|
Array.wrap(uploaded_files).map do |file_id|
|
358
414
|
Hyrax::UploadedFile.find(file_id)
|
359
415
|
end
|
360
416
|
end
|
361
417
|
|
362
|
-
def uploaded_s3_files(remote_files:
|
418
|
+
def uploaded_s3_files(remote_files: [])
|
363
419
|
return [] if remote_files.blank?
|
364
420
|
|
365
421
|
s3_bucket_name = ENV.fetch("STAGING_AREA_S3_BUCKET", "comet-staging-area-#{Rails.env}")
|
@@ -371,6 +427,41 @@ module Bulkrax
|
|
371
427
|
end.compact
|
372
428
|
end
|
373
429
|
|
430
|
+
def uploaded_remote_files(remote_files: [])
|
431
|
+
remote_files.map do |r|
|
432
|
+
file_path = download_file(r["url"])
|
433
|
+
next unless file_path
|
434
|
+
|
435
|
+
create_uploaded_file(file_path, r["file_name"])
|
436
|
+
end.compact
|
437
|
+
end
|
438
|
+
|
439
|
+
def download_file(url)
|
440
|
+
require 'open-uri'
|
441
|
+
require 'tempfile'
|
442
|
+
|
443
|
+
begin
|
444
|
+
file = Tempfile.new
|
445
|
+
file.binmode
|
446
|
+
file.write(URI.open(url).read)
|
447
|
+
file.rewind
|
448
|
+
file.path
|
449
|
+
rescue => e
|
450
|
+
Rails.logger.debug "Failed to download file from #{url}: #{e.message}"
|
451
|
+
nil
|
452
|
+
end
|
453
|
+
end
|
454
|
+
|
455
|
+
def create_uploaded_file(file_path, file_name)
|
456
|
+
file = File.open(file_path)
|
457
|
+
uploaded_file = Hyrax::UploadedFile.create(file: file, user: @user, filename: file_name)
|
458
|
+
file.close
|
459
|
+
uploaded_file
|
460
|
+
rescue => e
|
461
|
+
Rails.logger.debug "Failed to create Hyrax::UploadedFile for #{file_name}: #{e.message}"
|
462
|
+
nil
|
463
|
+
end
|
464
|
+
|
374
465
|
# @Override Destroy existing files with Hyrax::Transactions
|
375
466
|
def destroy_existing_files
|
376
467
|
existing_files = Hyrax.custom_queries.find_child_file_sets(resource: object)
|
@@ -38,11 +38,13 @@ module Bulkrax
|
|
38
38
|
#
|
39
39
|
# @see https://github.com/scientist-softserv/louisville-hyku/commit/128a9ef
|
40
40
|
class_attribute :update_child_records_works_file_sets, default: false
|
41
|
+
class_attribute :max_failure_count, default: 5
|
41
42
|
|
42
43
|
include DynamicRecordLookup
|
43
44
|
|
44
45
|
queue_as Bulkrax.config.ingest_queue_name
|
45
46
|
|
47
|
+
attr_accessor :user, :importer_run, :errors
|
46
48
|
##
|
47
49
|
# @param parent_identifier [String] Work/Collection ID or Bulkrax::Entry source_identifiers
|
48
50
|
# @param importer_run [Bulkrax::ImporterRun] current importer run (needed to properly update counters)
|
@@ -54,9 +56,10 @@ module Bulkrax
|
|
54
56
|
# is the child in the relationship, and vice versa if a child_identifier is passed.
|
55
57
|
#
|
56
58
|
# rubocop:disable Metrics/MethodLength
|
57
|
-
def perform(parent_identifier:, importer_run_id:) # rubocop:disable Metrics/AbcSize
|
58
|
-
|
59
|
-
|
59
|
+
def perform(parent_identifier:, importer_run_id: nil, run_user: nil, failure_count: 0) # rubocop:disable Metrics/AbcSize
|
60
|
+
importer_run = Bulkrax::ImporterRun.find(importer_run_id) if importer_run_id
|
61
|
+
user = run_user || importer_run&.user
|
62
|
+
ability = Ability.new(user)
|
60
63
|
|
61
64
|
parent_entry, parent_record = find_record(parent_identifier, importer_run_id)
|
62
65
|
|
@@ -69,19 +72,21 @@ module Bulkrax
|
|
69
72
|
if parent_record
|
70
73
|
conditionally_acquire_lock_for(parent_record.id) do
|
71
74
|
ActiveRecord::Base.uncached do
|
72
|
-
Bulkrax::PendingRelationship.where(parent_id: parent_identifier
|
75
|
+
Bulkrax::PendingRelationship.where(parent_id: parent_identifier)
|
73
76
|
.ordered.find_each do |rel|
|
74
77
|
process(relationship: rel, importer_run_id: importer_run_id, parent_record: parent_record, ability: ability)
|
75
78
|
number_of_successes += 1
|
79
|
+
@parent_record_members_added = true
|
76
80
|
rescue => e
|
77
81
|
number_of_failures += 1
|
82
|
+
rel.set_status_info(e, importer_run)
|
78
83
|
errors << e
|
79
84
|
end
|
80
85
|
end
|
81
86
|
|
82
87
|
# save record if members were added
|
83
88
|
if @parent_record_members_added
|
84
|
-
Bulkrax.object_factory.save!(resource: parent_record, user:
|
89
|
+
Bulkrax.object_factory.save!(resource: parent_record, user: user)
|
85
90
|
Bulkrax.object_factory.publish(event: 'object.membership.updated', object: parent_record)
|
86
91
|
Bulkrax.object_factory.update_index(resources: @child_members_added)
|
87
92
|
end
|
@@ -104,10 +109,17 @@ module Bulkrax
|
|
104
109
|
# rubocop:enable Rails/SkipsModelValidations
|
105
110
|
|
106
111
|
parent_entry&.set_status_info(errors.last, importer_run)
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
112
|
+
failure_count += 1
|
113
|
+
|
114
|
+
if failure_count < max_failure_count
|
115
|
+
reschedule(
|
116
|
+
parent_identifier: parent_identifier,
|
117
|
+
importer_run_id: importer_run_id,
|
118
|
+
run_user: run_user,
|
119
|
+
failure_count: failure_count
|
120
|
+
)
|
121
|
+
end
|
122
|
+
return errors # stop current job from continuing to run after rescheduling
|
111
123
|
else
|
112
124
|
# rubocop:disable Rails/SkipsModelValidations
|
113
125
|
ImporterRun.update_counters(importer_run_id, processed_relationships: number_of_successes)
|
@@ -116,8 +128,6 @@ module Bulkrax
|
|
116
128
|
end
|
117
129
|
# rubocop:enable Metrics/MethodLength
|
118
130
|
|
119
|
-
attr_reader :importer_run
|
120
|
-
|
121
131
|
private
|
122
132
|
|
123
133
|
##
|
@@ -170,7 +180,7 @@ module Bulkrax
|
|
170
180
|
Bulkrax.object_factory.add_resource_to_collection(
|
171
181
|
collection: parent_record,
|
172
182
|
resource: child_record,
|
173
|
-
user:
|
183
|
+
user: user
|
174
184
|
)
|
175
185
|
end
|
176
186
|
|
@@ -183,11 +193,8 @@ module Bulkrax
|
|
183
193
|
)
|
184
194
|
end
|
185
195
|
|
186
|
-
def reschedule(
|
187
|
-
CreateRelationshipsJob.set(wait: 10.minutes).perform_later(
|
188
|
-
parent_identifier: parent_identifier,
|
189
|
-
importer_run_id: importer_run_id
|
190
|
-
)
|
196
|
+
def reschedule(**kargs)
|
197
|
+
CreateRelationshipsJob.set(wait: 10.minutes).perform_later(**kargs)
|
191
198
|
end
|
192
199
|
end
|
193
200
|
end
|
@@ -1,5 +1,23 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module Bulkrax
|
4
|
-
class DeleteFileSetJob < DeleteJob
|
4
|
+
class DeleteFileSetJob < DeleteJob
|
5
|
+
def perform(entry, importer_run)
|
6
|
+
file_set = entry.factory.find
|
7
|
+
if file_set
|
8
|
+
parent = file_set.parent
|
9
|
+
if parent&.respond_to?(:ordered_members)
|
10
|
+
om = parent.ordered_members.to_a
|
11
|
+
om.delete(file_set)
|
12
|
+
parent.ordered_members = om
|
13
|
+
elsif parent.respond_to?(:member_ids)
|
14
|
+
parent.member_ids.delete(file_set.id)
|
15
|
+
Hyrax.persister.save(resource: parent)
|
16
|
+
end
|
17
|
+
parent.save
|
18
|
+
end
|
19
|
+
|
20
|
+
super
|
21
|
+
end
|
22
|
+
end
|
5
23
|
end
|
@@ -18,6 +18,9 @@ module Bulkrax
|
|
18
18
|
|
19
19
|
delegate :create_parent_child_relationships, :valid_import?, :write_errored_entries_file, :visibility, to: :parser
|
20
20
|
|
21
|
+
after_save :set_last_imported_at_from_importer_run
|
22
|
+
after_save :set_next_import_at_from_importer_run
|
23
|
+
|
21
24
|
attr_accessor :only_updates, :file_style, :file
|
22
25
|
attr_writer :current_run
|
23
26
|
|
@@ -149,6 +152,18 @@ module Bulkrax
|
|
149
152
|
@seen ||= {}
|
150
153
|
end
|
151
154
|
|
155
|
+
def import_file_path
|
156
|
+
self.parser_fields['import_file_path']
|
157
|
+
end
|
158
|
+
|
159
|
+
def original_file?
|
160
|
+
import_file_path && File.exist?(import_file_path)
|
161
|
+
end
|
162
|
+
|
163
|
+
def original_file
|
164
|
+
import_file_path if original_file?
|
165
|
+
end
|
166
|
+
|
152
167
|
def replace_files
|
153
168
|
self.parser_fields['replace_files']
|
154
169
|
end
|
@@ -235,5 +250,27 @@ module Bulkrax
|
|
235
250
|
rescue
|
236
251
|
"#{self.id}_#{self.created_at.strftime('%Y%m%d%H%M%S')}"
|
237
252
|
end
|
253
|
+
|
254
|
+
private
|
255
|
+
|
256
|
+
# Adding this here since we can update the importer without running the importer.
|
257
|
+
# When we simply save the importer (as in just updating the importer from the options),
|
258
|
+
# it does not trigger the after_save callback in the importer_run.
|
259
|
+
def set_last_imported_at_from_importer_run
|
260
|
+
return if @skip_set_last_imported_at # Prevent infinite loop
|
261
|
+
|
262
|
+
@skip_set_last_imported_at = true
|
263
|
+
importer_runs.last&.set_last_imported_at
|
264
|
+
@skip_set_last_imported_at = false
|
265
|
+
end
|
266
|
+
|
267
|
+
# @see #set_last_imported_at_from_importer_run
|
268
|
+
def set_next_import_at_from_importer_run
|
269
|
+
return if @skip_set_next_import_at # Prevent infinite loop
|
270
|
+
|
271
|
+
@skip_set_next_import_at = true
|
272
|
+
importer_runs.last&.set_next_import_at
|
273
|
+
@skip_set_next_import_at = false
|
274
|
+
end
|
238
275
|
end
|
239
276
|
end
|
@@ -6,6 +6,9 @@ module Bulkrax
|
|
6
6
|
has_many :statuses, as: :runnable, dependent: :destroy
|
7
7
|
has_many :pending_relationships, dependent: :destroy
|
8
8
|
|
9
|
+
after_save :set_last_imported_at
|
10
|
+
after_save :set_next_import_at
|
11
|
+
|
9
12
|
def parents
|
10
13
|
pending_relationships.pluck(:parent_id).uniq
|
11
14
|
end
|
@@ -15,5 +18,13 @@ module Bulkrax
|
|
15
18
|
# fallback to the configured user.
|
16
19
|
importer.user || Bulkrax.fallback_user_for_importer_exporter_processing
|
17
20
|
end
|
21
|
+
|
22
|
+
def set_last_imported_at
|
23
|
+
importer.update(last_imported_at: importer.last_imported_at)
|
24
|
+
end
|
25
|
+
|
26
|
+
def set_next_import_at
|
27
|
+
importer.update(next_import_at: importer.next_import_at)
|
28
|
+
end
|
18
29
|
end
|
19
30
|
end
|
@@ -150,7 +150,7 @@ module Bulkrax
|
|
150
150
|
end
|
151
151
|
end
|
152
152
|
|
153
|
-
# The visibility of the record. Acceptable values are: "open", "
|
153
|
+
# The visibility of the record. Acceptable values are: "open", "embargo", "lease", "authenticated", "restricted". The default is "open"
|
154
154
|
#
|
155
155
|
# @return [String]
|
156
156
|
# @see https://github.com/samvera/hydra-head/blob/main/hydra-access-controls/app/models/concerns/hydra/access_controls/access_right.rb Hydra::AccessControls::AccessRight for details on the range of values.
|
@@ -1,13 +1,15 @@
|
|
1
1
|
<div class='csv_fields'>
|
2
2
|
|
3
3
|
<%= fi.input :visibility,
|
4
|
+
label: 'Default Visibility',
|
4
5
|
collection: [
|
5
6
|
['Public', 'open'],
|
6
7
|
['Private', 'restricted'],
|
7
8
|
['Institution', 'authenticated']
|
8
9
|
],
|
9
10
|
selected: importer.parser_fields['visibility'] || 'open',
|
10
|
-
input_html: { class: 'form-control' }
|
11
|
+
input_html: { class: 'form-control' },
|
12
|
+
hint: 'If your CSV includes the visibility field, it will override the default setting.'
|
11
13
|
%>
|
12
14
|
|
13
15
|
<% if defined?(::Hyrax) %>
|
@@ -1,12 +1,12 @@
|
|
1
1
|
<div class="col-xs-12 main-header">
|
2
2
|
<h1><span class="fa fa-cloud-upload" aria-hidden="true"></span> Importer: <%= @importer.name %></h1>
|
3
|
-
|
4
|
-
|
5
|
-
|
3
|
+
<div class="pull-right">
|
4
|
+
<%= link_to 'Download Original File', importer_original_file_path(@importer.id), class: 'btn btn-primary', data: { turbolinks: false } if @importer.original_file %>
|
5
|
+
<% if @importer.failed_entries? %>
|
6
6
|
<%= link_to 'Export Errored Entries', importer_export_errors_path(@importer.id), class: 'btn btn-primary', data: { turbolinks: false }%>
|
7
7
|
<%= link_to 'Upload Corrected Entries', importer_upload_corrected_entries_path(@importer.id), class: 'btn btn-primary' if @importer.parser.is_a?(Bulkrax::CsvParser) %>
|
8
|
-
|
9
|
-
|
8
|
+
<% end %>
|
9
|
+
</div>
|
10
10
|
</div>
|
11
11
|
<div class="panel panel-default bulkrax-align-text">
|
12
12
|
<div class="panel-body">
|
data/config/routes.rb
CHANGED
data/lib/bulkrax/version.rb
CHANGED
@@ -141,4 +141,25 @@ namespace :bulkrax do
|
|
141
141
|
rescue => e
|
142
142
|
puts "(#{e.message})"
|
143
143
|
end
|
144
|
+
|
145
|
+
desc "Resave importers"
|
146
|
+
task resave_importers: :environment do
|
147
|
+
if defined?(::Hyku)
|
148
|
+
Account.find_each do |account|
|
149
|
+
next if account.name == "search"
|
150
|
+
switch!(account)
|
151
|
+
puts "=============== updating #{account.name} ============"
|
152
|
+
|
153
|
+
resave_importers
|
154
|
+
|
155
|
+
puts "=============== finished updating #{account.name} ============"
|
156
|
+
end
|
157
|
+
else
|
158
|
+
resave_importers
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
def resave_importers
|
163
|
+
Bulkrax::Importer.find_each(&:save!)
|
164
|
+
end
|
144
165
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bulkrax
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 8.
|
4
|
+
version: 8.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rob Kaufman
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-09-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|
@@ -345,7 +345,6 @@ files:
|
|
345
345
|
- app/factories/bulkrax/object_factory.rb
|
346
346
|
- app/factories/bulkrax/object_factory_interface.rb
|
347
347
|
- app/factories/bulkrax/valkyrie_object_factory.rb
|
348
|
-
- app/factories/bulkrax/valkyrize-hyku.code-workspace
|
349
348
|
- app/helpers/bulkrax/application_helper.rb
|
350
349
|
- app/helpers/bulkrax/exporters_helper.rb
|
351
350
|
- app/helpers/bulkrax/importers_helper.rb
|
@@ -483,6 +482,10 @@ files:
|
|
483
482
|
- db/migrate/20240208005801_denormalize_status_message.rb
|
484
483
|
- db/migrate/20240209070952_update_identifier_index.rb
|
485
484
|
- db/migrate/20240307053156_add_index_to_metadata_bulkrax_identifier.rb
|
485
|
+
- db/migrate/20240806161142_add_file_name_to_uploaded_files.rb
|
486
|
+
- db/migrate/20240823173525_add_error_tracking_to_pending_relationships.rb
|
487
|
+
- db/migrate/20240916182737_add_last_imported_at_to_bulkrax_importers.rb
|
488
|
+
- db/migrate/20240916182823_add_next_import_at_to_bulkrax_importers.rb
|
486
489
|
- lib/bulkrax.rb
|
487
490
|
- lib/bulkrax/engine.rb
|
488
491
|
- lib/bulkrax/entry_spec_helper.rb
|
@@ -1,19 +0,0 @@
|
|
1
|
-
{
|
2
|
-
"folders": [
|
3
|
-
{
|
4
|
-
"path": "../../../../../hyku"
|
5
|
-
},
|
6
|
-
{
|
7
|
-
"path": "../../../../hyrax"
|
8
|
-
},
|
9
|
-
{
|
10
|
-
"path": "../../.."
|
11
|
-
},
|
12
|
-
{
|
13
|
-
"path": "../../../../iiif_print"
|
14
|
-
}
|
15
|
-
],
|
16
|
-
"settings": {
|
17
|
-
"github.copilot.inlineSuggest.enable": true
|
18
|
-
}
|
19
|
-
}
|