bulkrax 9.5.1 → 9.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,545 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Bulkrax
4
+ # An abstract class that establishes the API for Bulkrax's import and export parsing.
5
+ #
6
+ # @abstract Subclass the Bulkrax::ApplicationParser to create a parser that handles a specific format (e.g. CSV, Bagit, XML, etc).
7
+ class ApplicationParser # rubocop:disable Metrics/ClassLength
8
+ attr_accessor :importerexporter, :headers
9
+ alias importer importerexporter
10
+ alias exporter importerexporter
11
+ delegate :only_updates, :limit, :current_run, :errors, :mapping,
12
+ :seen, :increment_counters, :parser_fields, :user, :keys_without_numbers,
13
+ :key_without_numbers, :status, :set_status_info, :status_info, :status_at,
14
+ :exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
15
+ :zip?, :file?, :remove_and_rerun, :zip_file?,
16
+ to: :importerexporter
17
+
18
+ # @todo Convert to `class_attribute :parser_fiels, default: {}`
19
+ def self.parser_fields
20
+ {}
21
+ end
22
+
23
+ # @return [TrueClass,FalseClass] this parser does or does not support exports.
24
+ #
25
+ # @todo Convert to `class_attribute :export_supported, default: false, instance_predicate: true` and `self << class; alias export_supported? export_supported; end`
26
+ def self.export_supported?
27
+ false
28
+ end
29
+
30
+ # @return [TrueClass,FalseClass] this parser does or does not support imports.
31
+ #
32
+ # @todo Convert to `class_attribute :import_supported, default: false, instance_predicate: true` and `self << class; alias import_supported? import_supported; end`
33
+ def self.import_supported?
34
+ true
35
+ end
36
+
37
+ def initialize(importerexporter)
38
+ @importerexporter = importerexporter
39
+ @headers = []
40
+ end
41
+
42
+ # @api public
43
+ # @abstract Subclass and override {#entry_class} to implement behavior for the parser.
44
+ def entry_class
45
+ raise NotImplementedError, 'must be defined'
46
+ end
47
+
48
+ def work_entry_class
49
+ entry_class
50
+ end
51
+
52
+ # @api public
53
+ # @abstract Subclass and override {#collection_entry_class} to implement behavior for the parser.
54
+ def collection_entry_class
55
+ raise NotImplementedError, 'must be defined'
56
+ end
57
+
58
+ # @api public
59
+ # @abstract Subclass and override {#file_set_entry_class} to implement behavior for the parser.
60
+ def file_set_entry_class
61
+ raise NotImplementedError, 'must be defined'
62
+ end
63
+
64
+ # @api public
65
+ # @abstract Subclass and override {#records} to implement behavior for the parser.
66
+ def records(_opts = {})
67
+ raise NotImplementedError, 'must be defined'
68
+ end
69
+
70
+ # @return [Symbol] the name of the identifying property in the source system from which we're
71
+ # importing (e.g. is *not* this application that mounts *this* Bulkrax engine).
72
+ #
73
+ # @see #work_identifier
74
+ # @see https://github.com/samvera-labs/bulkrax/wiki/CSV-Importer#source-identifier Bulkrax Wiki regarding source identifier
75
+ def source_identifier
76
+ @source_identifier ||= get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('from')&.first&.to_sym || :source_identifier
77
+ end
78
+
79
+ # @return [Symbol] the name of the identifying property for the system which we're importing
80
+ # into (e.g. the application that mounts *this* Bulkrax engine)
81
+ # @see #source_identifier
82
+ def work_identifier
83
+ @work_identifier ||= get_field_mapping_hash_for('source_identifier')&.keys&.first&.to_sym || :source
84
+ end
85
+
86
+ # @return [Symbol] the solr property of the source_identifier. Used for searching.
87
+ # defaults to work_identifier value + "_sim"
88
+ # @see #work_identifier
89
+ def work_identifier_search_field
90
+ @work_identifier_search_field ||= Array.wrap(get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('search_field'))&.first&.to_s || "#{work_identifier}_sim"
91
+ end
92
+
93
+ # @return [String]
94
+ def generated_metadata_mapping
95
+ @generated_metadata_mapping ||= 'generated'
96
+ end
97
+
98
+ # @return [String, NilClass]
99
+ # @see #related_parents_raw_mapping
100
+ def related_parents_raw_mapping
101
+ @related_parents_raw_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.values&.first&.[]('from')&.first
102
+ end
103
+
104
+ # @return [String]
105
+ # @see #related_parents_field_mapping
106
+ def related_parents_parsed_mapping
107
+ @related_parents_parsed_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.keys&.first || 'parents'
108
+ end
109
+
110
+ # @return [String, NilClass]
111
+ # @see #related_children_parsed_mapping
112
+ def related_children_raw_mapping
113
+ @related_children_raw_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.values&.first&.[]('from')&.first
114
+ end
115
+
116
+ # @return [String]
117
+ # @see #related_children_raw_mapping
118
+ def related_children_parsed_mapping
119
+ @related_children_parsed_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.keys&.first || 'children'
120
+ end
121
+
122
+ # @api private
123
+ def get_field_mapping_hash_for(key)
124
+ return instance_variable_get("@#{key}_hash") if instance_variable_get("@#{key}_hash").present?
125
+
126
+ mapping = importerexporter.field_mapping.is_a?(Hash) ? importerexporter.field_mapping : {}
127
+ instance_variable_set(
128
+ "@#{key}_hash",
129
+ mapping&.with_indifferent_access&.select { |_, h| h.key?(key) }
130
+ )
131
+ raise StandardError, "more than one #{key} declared: #{instance_variable_get("@#{key}_hash").keys.join(', ')}" if instance_variable_get("@#{key}_hash").length > 1
132
+
133
+ instance_variable_get("@#{key}_hash")
134
+ end
135
+
136
+ # @return [Array<String>]
137
+ def model_field_mappings
138
+ model_mappings = Bulkrax.field_mappings[self.class.to_s]&.dig('model', :from) || []
139
+ model_mappings |= ['model']
140
+
141
+ model_mappings
142
+ end
143
+
144
+ # @return [String]
145
+ def perform_method
146
+ if self.validate_only
147
+ 'perform_now'
148
+ else
149
+ 'perform_later'
150
+ end
151
+ end
152
+
153
+ # The visibility of the record. Acceptable values are: "open", "embargo", "lease", "authenticated", "restricted". The default is "open"
154
+ #
155
+ # @return [String]
156
+ # @see https://github.com/samvera/hydra-head/blob/main/hydra-access-controls/app/models/concerns/hydra/access_controls/access_right.rb Hydra::AccessControls::AccessRight for details on the range of values.
157
+ # @see https://github.com/samvera/hyrax/blob/bd2bcffc33e183904be2c175367648815f25bc2b/app/services/hyrax/visibility_intention.rb Hyrax::VisibilityIntention for how we process the visibility.
158
+ def visibility
159
+ @visibility ||= self.parser_fields['visibility'] || 'open'
160
+ end
161
+
162
+ def create_collections
163
+ create_objects(['collection'])
164
+ end
165
+
166
+ def create_works
167
+ create_objects(['work'])
168
+ end
169
+
170
+ def create_file_sets
171
+ create_objects(['file_set'])
172
+ end
173
+
174
+ def create_relationships
175
+ create_objects(['relationship'])
176
+ end
177
+
178
+ # @api public
179
+ #
180
+ # @param types [Array<Symbol>] the types of objects that we'll create.
181
+ #
182
+ # @see Bulkrax::Importer::DEFAULT_OBJECT_TYPES
183
+ # @see #create_collections
184
+ # @see #create_works
185
+ # @see #create_file_sets
186
+ # @see #create_relationships
187
+ def create_objects(types_array = nil)
188
+ index = 0
189
+ (types_array || %w[collection work file_set relationship]).each do |type|
190
+ if type.eql?('relationship')
191
+ ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: importerexporter.id)
192
+ next
193
+ end
194
+ send(type.pluralize).each do |current_record|
195
+ next unless record_has_source_identifier(current_record, index)
196
+ break if limit_reached?(limit, index)
197
+ seen[current_record[source_identifier]] = true
198
+ create_entry_and_job(current_record, type)
199
+ increment_counters(index, "#{type}": true)
200
+ index += 1
201
+ end
202
+ importer.record_status
203
+ end
204
+ true
205
+ rescue StandardError => e
206
+ set_status_info(e)
207
+ end
208
+
209
+ def rebuild_entries(types_array = nil)
210
+ index = 0
211
+ (types_array || %w[collection work file_set relationship]).each do |type|
212
+ # works are not guaranteed to have Work in the type
213
+ if type.eql?('relationship')
214
+ ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: importerexporter.id)
215
+ next
216
+ end
217
+ importer.entries.where(rebuild_entry_query(type, parser_fields['entry_statuses'])).find_each do |e|
218
+ seen[e.identifier] = true
219
+ e.status_info('Pending', importer.current_run)
220
+ if remove_and_rerun
221
+ delay = calculate_type_delay(type)
222
+ "Bulkrax::DeleteAndImport#{type.camelize}Job".constantize.set(wait: delay).send(perform_method, e, current_run)
223
+ else
224
+ "Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, e.id, current_run.id)
225
+ end
226
+ increment_counters(index)
227
+ index += 1
228
+ end
229
+ end
230
+ end
231
+
232
+ def rebuild_entry_query(type, statuses)
233
+ type_col = Bulkrax::Entry.arel_table['type']
234
+ status_col = Bulkrax::Entry.arel_table['status_message']
235
+
236
+ query = (type == 'work' ? type_col.does_not_match_all(%w[collection file_set]) : type_col.matches(type.camelize))
237
+ query.and(status_col.in(statuses))
238
+ end
239
+
240
+ def calculate_type_delay(type)
241
+ return 2.minutes if type == 'file_set'
242
+ return 1.minute if type == 'work'
243
+ return 0
244
+ end
245
+
246
+ def record_raw_metadata(record)
247
+ record.to_h
248
+ end
249
+
250
+ def record_deleted?(record)
251
+ return false unless record.key?(:delete)
252
+ ActiveModel::Type::Boolean.new.cast(record[:delete])
253
+ end
254
+
255
+ def record_remove_and_rerun?(record)
256
+ return false unless record.key?(:remove_and_rerun)
257
+ ActiveModel::Type::Boolean.new.cast(record[:remove_and_rerun])
258
+ end
259
+
260
+ def create_entry_and_job(current_record, type, identifier = nil)
261
+ identifier ||= current_record[source_identifier]
262
+ new_entry = find_or_create_entry(send("#{type}_entry_class"),
263
+ identifier,
264
+ 'Bulkrax::Importer',
265
+ record_raw_metadata(current_record))
266
+ new_entry.status_info('Pending', importer.current_run)
267
+ if record_deleted?(current_record)
268
+ "Bulkrax::Delete#{type.camelize}Job".constantize.send(perform_method, new_entry, current_run)
269
+ elsif record_remove_and_rerun?(current_record) || remove_and_rerun
270
+ delay = calculate_type_delay(type)
271
+ "Bulkrax::DeleteAndImport#{type.camelize}Job".constantize.set(wait: delay).send(perform_method, new_entry, current_run)
272
+ else
273
+ "Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, new_entry.id, current_run.id)
274
+ end
275
+ end
276
+
277
+ # Optional, define if using browse everything for file upload
278
+ def retrieve_cloud_files(_files, _importer); end
279
+
280
+ # @param file [#path, #original_filename] the file object that with the relevant data for the
281
+ # import.
282
+ def write_import_file(file)
283
+ path = File.join(path_for_import, file.original_filename)
284
+ FileUtils.mv(
285
+ file.path,
286
+ path
287
+ )
288
+ path
289
+ end
290
+
291
+ # Base path for imported and exported files
292
+ # @param [String]
293
+ # @return [String] the base path for files that this parser will "parse"
294
+ def base_path(type = 'import')
295
+ # account for multiple versions of hyku
296
+ is_multitenant = ENV['HYKU_MULTITENANT'] == 'true' || ENV['SETTINGS__MULTITENANCY__ENABLED'] == 'true'
297
+ is_multitenant ? File.join(Bulkrax.send("#{type}_path"), ::Site.instance.account.name) : Bulkrax.send("#{type}_path")
298
+ end
299
+
300
+ # Path where we'll store the import metadata and files
301
+ # this is used for uploaded and cloud files
302
+ # @return [String]
303
+ def path_for_import
304
+ @path_for_import = File.join(base_path, importerexporter.path_string)
305
+ FileUtils.mkdir_p(@path_for_import) unless File.exist?(@path_for_import)
306
+ @path_for_import
307
+ end
308
+
309
+ # @abstract Subclass and override {#setup_export_file} to implement behavior for the parser.
310
+ def setup_export_file
311
+ raise NotImplementedError, 'must be defined' if exporter?
312
+ end
313
+
314
+ # @abstract Subclass and override {#write_files} to implement behavior for the parser.
315
+ def write_files
316
+ raise NotImplementedError, 'must be defined' if exporter?
317
+ end
318
+
319
+ # @return [TrueClass,FalseClass]
320
+ def importer?
321
+ importerexporter.is_a?(Bulkrax::Importer)
322
+ end
323
+
324
+ # @return [TrueClass,FalseClass]
325
+ def exporter?
326
+ importerexporter.is_a?(Bulkrax::Exporter)
327
+ end
328
+
329
+ # @param limit [Integer] limit set on the importerexporter
330
+ # @param index [Integer] index of current iteration
331
+ # @return [TrueClass,FalseClass]
332
+ def limit_reached?(limit, index)
333
+ return false if limit.nil? || limit.zero? # no limit
334
+ index >= limit
335
+ end
336
+
337
+ # Override to add specific validations
338
+ # @return [TrueClass,FalseClass]
339
+ def valid_import?
340
+ true
341
+ end
342
+
343
+ # @return [TrueClass,FalseClass]
344
+ def record_has_source_identifier(record, index)
345
+ if record[source_identifier].blank?
346
+ if Bulkrax.fill_in_blank_source_identifiers.present?
347
+ record[source_identifier] = Bulkrax.fill_in_blank_source_identifiers.call(self, index)
348
+ else
349
+ invalid_record("Missing #{source_identifier} for #{record.to_h}\n")
350
+ false
351
+ end
352
+ else
353
+ true
354
+ end
355
+ end
356
+
357
+ # rubocop:disable Rails/SkipsModelValidations
358
+ def invalid_record(message)
359
+ current_run.invalid_records ||= ""
360
+ current_run.invalid_records += message
361
+ current_run.save
362
+ ImporterRun.increment_counter(:failed_records, current_run.id)
363
+ ImporterRun.decrement_counter(:enqueued_records, current_run.id) unless ImporterRun.find(current_run.id).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
364
+ end
365
+ # rubocop:enable Rails/SkipsModelValidations
366
+
367
+ # @return [Array<String>]
368
+ def required_elements
369
+ matched_elements = ((importerexporter.mapping.keys || []) & (Bulkrax.required_elements || []))
370
+ unless matched_elements.count == Bulkrax.required_elements.count
371
+ missing_elements = Bulkrax.required_elements - matched_elements
372
+ error_alert = "Missing mapping for at least one required element, missing mappings are: #{missing_elements.join(', ')}"
373
+ raise StandardError, error_alert
374
+ end
375
+ if Bulkrax.fill_in_blank_source_identifiers
376
+ Bulkrax.required_elements
377
+ else
378
+ Bulkrax.required_elements + [source_identifier]
379
+ end
380
+ end
381
+
382
+ def new_entry(entryclass, type)
383
+ entryclass.new(
384
+ importerexporter_id: importerexporter.id,
385
+ importerexporter_type: type
386
+ )
387
+ end
388
+
389
+ def find_or_create_entry(entryclass, identifier, type, raw_metadata = nil)
390
+ # limit entry search to just this importer or exporter. Don't go moving them
391
+ entry = importerexporter.entries.where(
392
+ identifier: identifier
393
+ ).first
394
+ entry ||= entryclass.new(
395
+ importerexporter_id: importerexporter.id,
396
+ importerexporter_type: type,
397
+ identifier: identifier
398
+ )
399
+ entry.raw_metadata = raw_metadata
400
+ # Setting parsed_metadata specifically for the id so we can find the object via the
401
+ # id in a delete. This is likely to get clobbered in a regular import, which is fine.
402
+ entry.parsed_metadata = { id: raw_metadata['id'] } if raw_metadata&.key?('id')
403
+ entry.save!
404
+ entry
405
+ end
406
+
407
+ # @todo - review this method - is it ever used?
408
+ def record(identifier, _opts = {})
409
+ return @record if @record
410
+
411
+ @record = entry_class.new(self, identifier)
412
+ @record.build
413
+ return @record
414
+ end
415
+
416
+ def total
417
+ 0
418
+ end
419
+
420
+ def collections_total
421
+ 0
422
+ end
423
+
424
+ def file_sets_total
425
+ 0
426
+ end
427
+
428
+ def write
429
+ write_files
430
+ zip
431
+ end
432
+
433
+ # Extracts a zip verbatim into {#importer_unzip_path}, preserving the zip's
434
+ # internal structure. Filters macOS junk (`__MACOSX/`, `.DS_Store`, `._*`).
435
+ # Parser subclasses that need to interpret the zip's structure (e.g.
436
+ # {Bulkrax::CsvParser#unzip_with_primary_csv}) should call a more specific
437
+ # method rather than this one.
438
+ def unzip(file_to_unzip)
439
+ return untar(file_to_unzip) if file_to_unzip.end_with?('.tar.gz')
440
+
441
+ dest_dir = importer_unzip_path(mkdir: true)
442
+ Zip::File.open(file_to_unzip) do |zip_file|
443
+ zip_file.each do |entry|
444
+ next unless entry.file?
445
+ next if macos_junk_entry?(entry.name)
446
+ reject_unsafe_entry!(entry.name)
447
+ dest_path = safe_extract_path(dest_dir, entry.name)
448
+ FileUtils.mkdir_p(File.dirname(dest_path))
449
+ next if File.exist?(dest_path)
450
+ extract_zip_entry(zip_file, entry, dest_dir, entry.name, dest_path)
451
+ end
452
+ end
453
+ end
454
+
455
+ # rubyzip 2.x: extract(entry, absolute_dest_path)
456
+ # rubyzip 3.x: extract(entry, relative_name, destination_directory: dir)
457
+ #
458
+ # Callers are responsible for passing a `dest_path` produced by
459
+ # {#safe_extract_path} so the write can't escape `dest_dir`.
460
+ def extract_zip_entry(zip_file, entry, dest_dir, relative_name, dest_path)
461
+ if zip_file.method(:extract).arity == 2
462
+ zip_file.extract(entry, dest_path)
463
+ else
464
+ zip_file.extract(entry, relative_name, destination_directory: dest_dir)
465
+ end
466
+ end
467
+
468
+ def macos_junk_entry?(name)
469
+ name.start_with?('__MACOSX/') || name.split('/').any? { |part| part == '.DS_Store' || part.start_with?('._') }
470
+ end
471
+
472
+ # Zip Slip preflight — reject entries whose names are obviously unsafe
473
+ # (absolute paths, `..` segments) before we touch the filesystem.
474
+ # {#safe_extract_path} is the final line of defense; this check just
475
+ # fails fast with a clear message. Delegates to {Bulkrax::SafePath} so
476
+ # cloud-file destinations and sources share the same rules.
477
+ #
478
+ # @raise [Bulkrax::UnzipError] if the entry name is unsafe
479
+ <<<<<<< HEAD
480
+ def reject_unsafe_entry!(name)
481
+ return unless name.start_with?('/') || name.split('/').include?('..')
482
+ raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: name)
483
+ =======
484
+ def self.reject_unsafe_entry!(name)
485
+ Bulkrax::SafePath.reject_unsafe_entry!(name)
486
+ rescue Bulkrax::UnsafePathError
487
+ raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: name.dup.force_encoding(Encoding::UTF_8).scrub)
488
+ >>>>>>> acc18e45 (Merge commit from fork)
489
+ end
490
+
491
+ # Zip Slip chokepoint. Resolves `relative_dest` against `dest_dir` and
492
+ # returns the absolute destination path — but only if it stays inside
493
+ # `dest_dir`, including through any symlink. Callers must use this
494
+ # value rather than building their own path with `File.join`, so the
495
+ # path returned is always safe by construction.
496
+ #
497
+ # @return [String] absolute destination path, validated to be inside `dest_dir`
498
+ # @raise [Bulkrax::UnzipError] if `relative_dest` escapes `dest_dir`
499
+ def safe_extract_path(dest_dir, relative_dest)
500
+ Bulkrax::SafePath.confine!(dest_dir, relative_dest)
501
+ rescue Bulkrax::UnsafePathError
502
+ raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: relative_dest)
503
+ end
504
+
505
+ def copy_file(file_to_copy)
506
+ destination = File.join(importer_unzip_path(mkdir: true), File.basename(file_to_copy))
507
+ FileUtils.cp(file_to_copy, destination)
508
+ end
509
+
510
+ def untar(file_to_untar)
511
+ Dir.mkdir(importer_unzip_path(mkdir: true)) unless File.directory?(importer_unzip_path(mkdir: true))
512
+ command = "tar -xzf #{Shellwords.escape(file_to_untar)} -C #{Shellwords.escape(importer_unzip_path)}"
513
+ result = system(command)
514
+ raise "Failed to extract #{file_to_untar}" unless result
515
+ end
516
+
517
+ def zip
518
+ FileUtils.mkdir_p(exporter_export_zip_path)
519
+
520
+ Dir["#{exporter_export_path}/**"].each do |folder|
521
+ zip_path = "#{exporter_export_zip_path.split('/').last}_#{folder.split('/').last}.zip"
522
+ FileUtils.rm_rf("#{exporter_export_zip_path}/#{zip_path}")
523
+
524
+ Zip::File.open(File.join("#{exporter_export_zip_path}/#{zip_path}"), create: true) do |zip_file|
525
+ Dir["#{folder}/**/**"].each do |file|
526
+ zip_file.add(file.sub("#{folder}/", ''), file)
527
+ end
528
+ end
529
+ end
530
+ end
531
+
532
+ # Path for the import
533
+ # @return [String]
534
+ def import_file_path
535
+ @import_file_path ||= real_import_file_path
536
+ end
537
+
538
+ private
539
+
540
+ # @return [String]
541
+ def real_import_file_path
542
+ parser_fields['import_file_path']
543
+ end
544
+ end
545
+ end
@@ -178,12 +178,19 @@ unless ENV.fetch('BULKRAX_NO_BAGIT', 'false').to_s == 'true'
178
178
  # @todo - investigate getting directory structure
179
179
  # @todo - investigate using perform_later, and having the importer check for
180
180
  # DownloadCloudFileJob before it starts
181
- def retrieve_cloud_files(files, _importer)
181
+ def retrieve_cloud_files(files, importer)
182
182
  # There should only be one zip file for Bagit, take the first
183
183
  return if files['0'].blank?
184
- target_file = File.join(path_for_import, files['0']['file_name'].tr(' ', '_'))
184
+
185
+ file = files['0']
186
+ # Checked on the raw file_name before any normalization — see
187
+ # Bulkrax::CsvParser#plan_cloud_file for why this must come first.
188
+ Bulkrax::CloudSourceValidator.preflight!(file)
189
+
190
+ relative_filename = file['file_name'].tr(' ', '_')
191
+ target_file = Bulkrax::SafePath.confine!(path_for_import, relative_filename, require_file: true)
185
192
  # Now because we want the files in place before the importer runs
186
- Bulkrax::DownloadCloudFileJob.perform_now(files['0'], target_file)
193
+ Bulkrax::DownloadCloudFileJob.perform_now(file, importer_id: importer.id, relative_filename: relative_filename)
187
194
  return target_file
188
195
  end
189
196
 
@@ -209,24 +209,55 @@ module Bulkrax
209
209
  # @todo - investigate using perform_later, and having the importer check for
210
210
  # DownloadCloudFileJob before it starts
211
211
  def retrieve_cloud_files(files, importer)
212
+ plan = plan_cloud_files(files)
213
+ dispatch_cloud_files!(plan, importer)
214
+ importer[:parser_fields]['original_file_paths'] = plan.map { |_file, _relative_filename, target_file| target_file }
215
+ return nil
216
+ end
217
+
218
+ # Validates and resolves the whole batch, without enqueueing anything
219
+ # yet. Otherwise an earlier, valid file's job would already be running
220
+ # by the time a later file in the same batch is found to be unsafe.
221
+ # Split out from {#retrieve_cloud_files} so a caller (the importers
222
+ # controller) can persist the resolved paths before dispatching.
223
+ #
224
+ # @return [Array<Array(Hash, String, String)>] one entry per file: the
225
+ # file spec, its relative destination filename, and the resolved
226
+ # absolute target path
227
+ def plan_cloud_files(files)
212
228
  files_path = File.join(path_for_import, 'files')
213
229
  FileUtils.mkdir_p(files_path) unless File.exist?(files_path)
214
- target_files = []
215
- files.each_pair do |_key, file|
216
- # fixes bug where auth headers do not get attached properly
217
- if file['auth_header'].present?
218
- file['headers'] ||= {}
219
- file['headers'].merge!(file['auth_header'])
220
- end
221
- # this only works for uniquely named files
222
- target_file = File.join(files_path, file['file_name'].tr(' ', '_'))
223
- target_files << target_file
224
- # Now because we want the files in place before the importer runs
225
- # Problematic for a large upload
226
- Bulkrax::DownloadCloudFileJob.perform_later(file, target_file)
230
+
231
+ files.values.map { |file| plan_cloud_file(file) }
232
+ end
233
+
234
+ # Enqueues a download job for each entry of an already-validated plan
235
+ # (see {#plan_cloud_files}).
236
+ def dispatch_cloud_files!(plan, importer)
237
+ plan.each { |file, relative_filename, _target_file| Bulkrax::DownloadCloudFileJob.perform_later(file, importer_id: importer.id, relative_filename: relative_filename) }
238
+ end
239
+
240
+ # Validates one cloud-file spec and resolves its destination, without
241
+ # enqueueing anything yet.
242
+ #
243
+ # @return [Array(Hash, String, String)] the file spec, its relative
244
+ # destination filename, and the resolved absolute target path
245
+ def plan_cloud_file(file)
246
+ # fixes bug where auth headers do not get attached properly
247
+ if file['auth_header'].present?
248
+ file['headers'] ||= {}
249
+ file['headers'].merge!(file['auth_header'])
227
250
  end
228
- importer[:parser_fields]['original_file_paths'] = target_files
229
- return nil
251
+
252
+ # Checked on the raw file_name, before it's joined under 'files/':
253
+ # that join would otherwise neutralize a leading `/` (File.join
254
+ # doesn't treat it as absolute once it's not the first argument).
255
+ Bulkrax::CloudSourceValidator.preflight!(file)
256
+
257
+ # this only works for uniquely named files
258
+ relative_filename = File.join('files', file['file_name'].tr(' ', '_'))
259
+ target_file = Bulkrax::SafePath.confine!(path_for_import, relative_filename, require_file: true)
260
+ [file, relative_filename, target_file]
230
261
  end
231
262
 
232
263
  # export methods