bulkrax 9.5.2 → 10.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +49 -0
  3. data/app/controllers/bulkrax/application_controller.rb +83 -0
  4. data/app/controllers/bulkrax/entries_controller.rb +16 -13
  5. data/app/controllers/bulkrax/exporters_controller.rb +21 -25
  6. data/app/controllers/bulkrax/guided_imports_controller.rb +4 -8
  7. data/app/controllers/bulkrax/importers_controller.rb +43 -87
  8. data/app/controllers/concerns/bulkrax/datatables_behavior.rb +18 -12
  9. data/app/controllers/concerns/bulkrax/importer_file_handler.rb +9 -55
  10. data/app/factories/bulkrax/object_factory.rb +2 -2
  11. data/app/factories/bulkrax/valkyrie_object_factory.rb +29 -17
  12. data/app/helpers/bulkrax/validation_helper.rb +1 -1
  13. data/app/jobs/bulkrax/create_relationships_job.rb +1 -1
  14. data/app/jobs/bulkrax/download_cloud_file_job.rb +21 -156
  15. data/app/jobs/bulkrax/import_collection_job.rb +9 -2
  16. data/app/matchers/bulkrax/application_matcher.rb +5 -2
  17. data/app/models/bulkrax/csv_entry.rb +1 -1
  18. data/app/models/bulkrax/current.rb +7 -0
  19. data/app/models/bulkrax/export_scope.rb +21 -0
  20. data/app/models/bulkrax/exporter.rb +2 -2
  21. data/app/models/bulkrax/importer.rb +1 -1
  22. data/app/models/concerns/bulkrax/ability_behavior.rb +139 -0
  23. data/app/models/concerns/bulkrax/has_matchers.rb +1 -1
  24. data/app/models/concerns/bulkrax/import_behavior.rb +3 -3
  25. data/app/parsers/bulkrax/application_parser.rb +10 -10
  26. data/app/parsers/bulkrax/bagit_parser.rb +3 -10
  27. data/app/parsers/bulkrax/csv_parser.rb +28 -58
  28. data/app/parsers/bulkrax/oai_dc_parser.rb +1 -1
  29. data/app/parsers/bulkrax/parser_export_record_set.rb +26 -8
  30. data/app/parsers/concerns/bulkrax/csv_parser/csv_template_generation.rb +1 -1
  31. data/app/parsers/concerns/bulkrax/csv_parser/csv_validation.rb +1 -2
  32. data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_helpers.rb +18 -1
  33. data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_hierarchy.rb +2 -2
  34. data/app/search_builders/bulkrax/export_search_builder.rb +9 -0
  35. data/app/services/bulkrax/csv_template/column_builder.rb +2 -2
  36. data/app/services/bulkrax/csv_template/mapping_manager.rb +9 -7
  37. data/app/services/bulkrax/csv_template/schema_analyzer.rb +1 -5
  38. data/config/locales/bulkrax.en.yml +0 -2
  39. data/lib/bulkrax/engine.rb +1 -1
  40. data/lib/bulkrax/version.rb +1 -1
  41. data/lib/bulkrax.rb +76 -11
  42. data/lib/generators/bulkrax/install_generator.rb +22 -10
  43. data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb +1 -1
  44. metadata +32 -34
  45. data/app/errors/bulkrax/unsafe_cloud_source_error.rb +0 -11
  46. data/app/errors/bulkrax/unsafe_path_error.rb +0 -8
  47. data/app/parsers/bulkrax/application_parser.rb.orig +0 -545
  48. data/app/services/bulkrax/cloud_source_validator.rb +0 -123
  49. data/app/services/bulkrax/safe_path.rb +0 -129
  50. data/lib/wings/custom_queries/find_by_source_identifier.rb +0 -41
@@ -1,545 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Bulkrax
4
- # An abstract class that establishes the API for Bulkrax's import and export parsing.
5
- #
6
- # @abstract Subclass the Bulkrax::ApplicationParser to create a parser that handles a specific format (e.g. CSV, Bagit, XML, etc).
7
- class ApplicationParser # rubocop:disable Metrics/ClassLength
8
- attr_accessor :importerexporter, :headers
9
- alias importer importerexporter
10
- alias exporter importerexporter
11
- delegate :only_updates, :limit, :current_run, :errors, :mapping,
12
- :seen, :increment_counters, :parser_fields, :user, :keys_without_numbers,
13
- :key_without_numbers, :status, :set_status_info, :status_info, :status_at,
14
- :exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
15
- :zip?, :file?, :remove_and_rerun, :zip_file?,
16
- to: :importerexporter
17
-
18
- # @todo Convert to `class_attribute :parser_fiels, default: {}`
19
- def self.parser_fields
20
- {}
21
- end
22
-
23
- # @return [TrueClass,FalseClass] this parser does or does not support exports.
24
- #
25
- # @todo Convert to `class_attribute :export_supported, default: false, instance_predicate: true` and `self << class; alias export_supported? export_supported; end`
26
- def self.export_supported?
27
- false
28
- end
29
-
30
- # @return [TrueClass,FalseClass] this parser does or does not support imports.
31
- #
32
- # @todo Convert to `class_attribute :import_supported, default: false, instance_predicate: true` and `self << class; alias import_supported? import_supported; end`
33
- def self.import_supported?
34
- true
35
- end
36
-
37
- def initialize(importerexporter)
38
- @importerexporter = importerexporter
39
- @headers = []
40
- end
41
-
42
- # @api public
43
- # @abstract Subclass and override {#entry_class} to implement behavior for the parser.
44
- def entry_class
45
- raise NotImplementedError, 'must be defined'
46
- end
47
-
48
- def work_entry_class
49
- entry_class
50
- end
51
-
52
- # @api public
53
- # @abstract Subclass and override {#collection_entry_class} to implement behavior for the parser.
54
- def collection_entry_class
55
- raise NotImplementedError, 'must be defined'
56
- end
57
-
58
- # @api public
59
- # @abstract Subclass and override {#file_set_entry_class} to implement behavior for the parser.
60
- def file_set_entry_class
61
- raise NotImplementedError, 'must be defined'
62
- end
63
-
64
- # @api public
65
- # @abstract Subclass and override {#records} to implement behavior for the parser.
66
- def records(_opts = {})
67
- raise NotImplementedError, 'must be defined'
68
- end
69
-
70
- # @return [Symbol] the name of the identifying property in the source system from which we're
71
- # importing (e.g. is *not* this application that mounts *this* Bulkrax engine).
72
- #
73
- # @see #work_identifier
74
- # @see https://github.com/samvera-labs/bulkrax/wiki/CSV-Importer#source-identifier Bulkrax Wiki regarding source identifier
75
- def source_identifier
76
- @source_identifier ||= get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('from')&.first&.to_sym || :source_identifier
77
- end
78
-
79
- # @return [Symbol] the name of the identifying property for the system which we're importing
80
- # into (e.g. the application that mounts *this* Bulkrax engine)
81
- # @see #source_identifier
82
- def work_identifier
83
- @work_identifier ||= get_field_mapping_hash_for('source_identifier')&.keys&.first&.to_sym || :source
84
- end
85
-
86
- # @return [Symbol] the solr property of the source_identifier. Used for searching.
87
- # defaults to work_identifier value + "_sim"
88
- # @see #work_identifier
89
- def work_identifier_search_field
90
- @work_identifier_search_field ||= Array.wrap(get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('search_field'))&.first&.to_s || "#{work_identifier}_sim"
91
- end
92
-
93
- # @return [String]
94
- def generated_metadata_mapping
95
- @generated_metadata_mapping ||= 'generated'
96
- end
97
-
98
- # @return [String, NilClass]
99
- # @see #related_parents_raw_mapping
100
- def related_parents_raw_mapping
101
- @related_parents_raw_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.values&.first&.[]('from')&.first
102
- end
103
-
104
- # @return [String]
105
- # @see #related_parents_field_mapping
106
- def related_parents_parsed_mapping
107
- @related_parents_parsed_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.keys&.first || 'parents'
108
- end
109
-
110
- # @return [String, NilClass]
111
- # @see #related_children_parsed_mapping
112
- def related_children_raw_mapping
113
- @related_children_raw_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.values&.first&.[]('from')&.first
114
- end
115
-
116
- # @return [String]
117
- # @see #related_children_raw_mapping
118
- def related_children_parsed_mapping
119
- @related_children_parsed_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.keys&.first || 'children'
120
- end
121
-
122
- # @api private
123
- def get_field_mapping_hash_for(key)
124
- return instance_variable_get("@#{key}_hash") if instance_variable_get("@#{key}_hash").present?
125
-
126
- mapping = importerexporter.field_mapping.is_a?(Hash) ? importerexporter.field_mapping : {}
127
- instance_variable_set(
128
- "@#{key}_hash",
129
- mapping&.with_indifferent_access&.select { |_, h| h.key?(key) }
130
- )
131
- raise StandardError, "more than one #{key} declared: #{instance_variable_get("@#{key}_hash").keys.join(', ')}" if instance_variable_get("@#{key}_hash").length > 1
132
-
133
- instance_variable_get("@#{key}_hash")
134
- end
135
-
136
- # @return [Array<String>]
137
- def model_field_mappings
138
- model_mappings = Bulkrax.field_mappings[self.class.to_s]&.dig('model', :from) || []
139
- model_mappings |= ['model']
140
-
141
- model_mappings
142
- end
143
-
144
- # @return [String]
145
- def perform_method
146
- if self.validate_only
147
- 'perform_now'
148
- else
149
- 'perform_later'
150
- end
151
- end
152
-
153
- # The visibility of the record. Acceptable values are: "open", "embargo", "lease", "authenticated", "restricted". The default is "open"
154
- #
155
- # @return [String]
156
- # @see https://github.com/samvera/hydra-head/blob/main/hydra-access-controls/app/models/concerns/hydra/access_controls/access_right.rb Hydra::AccessControls::AccessRight for details on the range of values.
157
- # @see https://github.com/samvera/hyrax/blob/bd2bcffc33e183904be2c175367648815f25bc2b/app/services/hyrax/visibility_intention.rb Hyrax::VisibilityIntention for how we process the visibility.
158
- def visibility
159
- @visibility ||= self.parser_fields['visibility'] || 'open'
160
- end
161
-
162
- def create_collections
163
- create_objects(['collection'])
164
- end
165
-
166
- def create_works
167
- create_objects(['work'])
168
- end
169
-
170
- def create_file_sets
171
- create_objects(['file_set'])
172
- end
173
-
174
- def create_relationships
175
- create_objects(['relationship'])
176
- end
177
-
178
- # @api public
179
- #
180
- # @param types [Array<Symbol>] the types of objects that we'll create.
181
- #
182
- # @see Bulkrax::Importer::DEFAULT_OBJECT_TYPES
183
- # @see #create_collections
184
- # @see #create_works
185
- # @see #create_file_sets
186
- # @see #create_relationships
187
- def create_objects(types_array = nil)
188
- index = 0
189
- (types_array || %w[collection work file_set relationship]).each do |type|
190
- if type.eql?('relationship')
191
- ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: importerexporter.id)
192
- next
193
- end
194
- send(type.pluralize).each do |current_record|
195
- next unless record_has_source_identifier(current_record, index)
196
- break if limit_reached?(limit, index)
197
- seen[current_record[source_identifier]] = true
198
- create_entry_and_job(current_record, type)
199
- increment_counters(index, "#{type}": true)
200
- index += 1
201
- end
202
- importer.record_status
203
- end
204
- true
205
- rescue StandardError => e
206
- set_status_info(e)
207
- end
208
-
209
- def rebuild_entries(types_array = nil)
210
- index = 0
211
- (types_array || %w[collection work file_set relationship]).each do |type|
212
- # works are not guaranteed to have Work in the type
213
- if type.eql?('relationship')
214
- ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: importerexporter.id)
215
- next
216
- end
217
- importer.entries.where(rebuild_entry_query(type, parser_fields['entry_statuses'])).find_each do |e|
218
- seen[e.identifier] = true
219
- e.status_info('Pending', importer.current_run)
220
- if remove_and_rerun
221
- delay = calculate_type_delay(type)
222
- "Bulkrax::DeleteAndImport#{type.camelize}Job".constantize.set(wait: delay).send(perform_method, e, current_run)
223
- else
224
- "Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, e.id, current_run.id)
225
- end
226
- increment_counters(index)
227
- index += 1
228
- end
229
- end
230
- end
231
-
232
- def rebuild_entry_query(type, statuses)
233
- type_col = Bulkrax::Entry.arel_table['type']
234
- status_col = Bulkrax::Entry.arel_table['status_message']
235
-
236
- query = (type == 'work' ? type_col.does_not_match_all(%w[collection file_set]) : type_col.matches(type.camelize))
237
- query.and(status_col.in(statuses))
238
- end
239
-
240
- def calculate_type_delay(type)
241
- return 2.minutes if type == 'file_set'
242
- return 1.minute if type == 'work'
243
- return 0
244
- end
245
-
246
- def record_raw_metadata(record)
247
- record.to_h
248
- end
249
-
250
- def record_deleted?(record)
251
- return false unless record.key?(:delete)
252
- ActiveModel::Type::Boolean.new.cast(record[:delete])
253
- end
254
-
255
- def record_remove_and_rerun?(record)
256
- return false unless record.key?(:remove_and_rerun)
257
- ActiveModel::Type::Boolean.new.cast(record[:remove_and_rerun])
258
- end
259
-
260
- def create_entry_and_job(current_record, type, identifier = nil)
261
- identifier ||= current_record[source_identifier]
262
- new_entry = find_or_create_entry(send("#{type}_entry_class"),
263
- identifier,
264
- 'Bulkrax::Importer',
265
- record_raw_metadata(current_record))
266
- new_entry.status_info('Pending', importer.current_run)
267
- if record_deleted?(current_record)
268
- "Bulkrax::Delete#{type.camelize}Job".constantize.send(perform_method, new_entry, current_run)
269
- elsif record_remove_and_rerun?(current_record) || remove_and_rerun
270
- delay = calculate_type_delay(type)
271
- "Bulkrax::DeleteAndImport#{type.camelize}Job".constantize.set(wait: delay).send(perform_method, new_entry, current_run)
272
- else
273
- "Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, new_entry.id, current_run.id)
274
- end
275
- end
276
-
277
- # Optional, define if using browse everything for file upload
278
- def retrieve_cloud_files(_files, _importer); end
279
-
280
- # @param file [#path, #original_filename] the file object that with the relevant data for the
281
- # import.
282
- def write_import_file(file)
283
- path = File.join(path_for_import, file.original_filename)
284
- FileUtils.mv(
285
- file.path,
286
- path
287
- )
288
- path
289
- end
290
-
291
- # Base path for imported and exported files
292
- # @param [String]
293
- # @return [String] the base path for files that this parser will "parse"
294
- def base_path(type = 'import')
295
- # account for multiple versions of hyku
296
- is_multitenant = ENV['HYKU_MULTITENANT'] == 'true' || ENV['SETTINGS__MULTITENANCY__ENABLED'] == 'true'
297
- is_multitenant ? File.join(Bulkrax.send("#{type}_path"), ::Site.instance.account.name) : Bulkrax.send("#{type}_path")
298
- end
299
-
300
- # Path where we'll store the import metadata and files
301
- # this is used for uploaded and cloud files
302
- # @return [String]
303
- def path_for_import
304
- @path_for_import = File.join(base_path, importerexporter.path_string)
305
- FileUtils.mkdir_p(@path_for_import) unless File.exist?(@path_for_import)
306
- @path_for_import
307
- end
308
-
309
- # @abstract Subclass and override {#setup_export_file} to implement behavior for the parser.
310
- def setup_export_file
311
- raise NotImplementedError, 'must be defined' if exporter?
312
- end
313
-
314
- # @abstract Subclass and override {#write_files} to implement behavior for the parser.
315
- def write_files
316
- raise NotImplementedError, 'must be defined' if exporter?
317
- end
318
-
319
- # @return [TrueClass,FalseClass]
320
- def importer?
321
- importerexporter.is_a?(Bulkrax::Importer)
322
- end
323
-
324
- # @return [TrueClass,FalseClass]
325
- def exporter?
326
- importerexporter.is_a?(Bulkrax::Exporter)
327
- end
328
-
329
- # @param limit [Integer] limit set on the importerexporter
330
- # @param index [Integer] index of current iteration
331
- # @return [TrueClass,FalseClass]
332
- def limit_reached?(limit, index)
333
- return false if limit.nil? || limit.zero? # no limit
334
- index >= limit
335
- end
336
-
337
- # Override to add specific validations
338
- # @return [TrueClass,FalseClass]
339
- def valid_import?
340
- true
341
- end
342
-
343
- # @return [TrueClass,FalseClass]
344
- def record_has_source_identifier(record, index)
345
- if record[source_identifier].blank?
346
- if Bulkrax.fill_in_blank_source_identifiers.present?
347
- record[source_identifier] = Bulkrax.fill_in_blank_source_identifiers.call(self, index)
348
- else
349
- invalid_record("Missing #{source_identifier} for #{record.to_h}\n")
350
- false
351
- end
352
- else
353
- true
354
- end
355
- end
356
-
357
- # rubocop:disable Rails/SkipsModelValidations
358
- def invalid_record(message)
359
- current_run.invalid_records ||= ""
360
- current_run.invalid_records += message
361
- current_run.save
362
- ImporterRun.increment_counter(:failed_records, current_run.id)
363
- ImporterRun.decrement_counter(:enqueued_records, current_run.id) unless ImporterRun.find(current_run.id).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
364
- end
365
- # rubocop:enable Rails/SkipsModelValidations
366
-
367
- # @return [Array<String>]
368
- def required_elements
369
- matched_elements = ((importerexporter.mapping.keys || []) & (Bulkrax.required_elements || []))
370
- unless matched_elements.count == Bulkrax.required_elements.count
371
- missing_elements = Bulkrax.required_elements - matched_elements
372
- error_alert = "Missing mapping for at least one required element, missing mappings are: #{missing_elements.join(', ')}"
373
- raise StandardError, error_alert
374
- end
375
- if Bulkrax.fill_in_blank_source_identifiers
376
- Bulkrax.required_elements
377
- else
378
- Bulkrax.required_elements + [source_identifier]
379
- end
380
- end
381
-
382
- def new_entry(entryclass, type)
383
- entryclass.new(
384
- importerexporter_id: importerexporter.id,
385
- importerexporter_type: type
386
- )
387
- end
388
-
389
- def find_or_create_entry(entryclass, identifier, type, raw_metadata = nil)
390
- # limit entry search to just this importer or exporter. Don't go moving them
391
- entry = importerexporter.entries.where(
392
- identifier: identifier
393
- ).first
394
- entry ||= entryclass.new(
395
- importerexporter_id: importerexporter.id,
396
- importerexporter_type: type,
397
- identifier: identifier
398
- )
399
- entry.raw_metadata = raw_metadata
400
- # Setting parsed_metadata specifically for the id so we can find the object via the
401
- # id in a delete. This is likely to get clobbered in a regular import, which is fine.
402
- entry.parsed_metadata = { id: raw_metadata['id'] } if raw_metadata&.key?('id')
403
- entry.save!
404
- entry
405
- end
406
-
407
- # @todo - review this method - is it ever used?
408
- def record(identifier, _opts = {})
409
- return @record if @record
410
-
411
- @record = entry_class.new(self, identifier)
412
- @record.build
413
- return @record
414
- end
415
-
416
- def total
417
- 0
418
- end
419
-
420
- def collections_total
421
- 0
422
- end
423
-
424
- def file_sets_total
425
- 0
426
- end
427
-
428
- def write
429
- write_files
430
- zip
431
- end
432
-
433
- # Extracts a zip verbatim into {#importer_unzip_path}, preserving the zip's
434
- # internal structure. Filters macOS junk (`__MACOSX/`, `.DS_Store`, `._*`).
435
- # Parser subclasses that need to interpret the zip's structure (e.g.
436
- # {Bulkrax::CsvParser#unzip_with_primary_csv}) should call a more specific
437
- # method rather than this one.
438
- def unzip(file_to_unzip)
439
- return untar(file_to_unzip) if file_to_unzip.end_with?('.tar.gz')
440
-
441
- dest_dir = importer_unzip_path(mkdir: true)
442
- Zip::File.open(file_to_unzip) do |zip_file|
443
- zip_file.each do |entry|
444
- next unless entry.file?
445
- next if macos_junk_entry?(entry.name)
446
- reject_unsafe_entry!(entry.name)
447
- dest_path = safe_extract_path(dest_dir, entry.name)
448
- FileUtils.mkdir_p(File.dirname(dest_path))
449
- next if File.exist?(dest_path)
450
- extract_zip_entry(zip_file, entry, dest_dir, entry.name, dest_path)
451
- end
452
- end
453
- end
454
-
455
- # rubyzip 2.x: extract(entry, absolute_dest_path)
456
- # rubyzip 3.x: extract(entry, relative_name, destination_directory: dir)
457
- #
458
- # Callers are responsible for passing a `dest_path` produced by
459
- # {#safe_extract_path} so the write can't escape `dest_dir`.
460
- def extract_zip_entry(zip_file, entry, dest_dir, relative_name, dest_path)
461
- if zip_file.method(:extract).arity == 2
462
- zip_file.extract(entry, dest_path)
463
- else
464
- zip_file.extract(entry, relative_name, destination_directory: dest_dir)
465
- end
466
- end
467
-
468
- def macos_junk_entry?(name)
469
- name.start_with?('__MACOSX/') || name.split('/').any? { |part| part == '.DS_Store' || part.start_with?('._') }
470
- end
471
-
472
- # Zip Slip preflight — reject entries whose names are obviously unsafe
473
- # (absolute paths, `..` segments) before we touch the filesystem.
474
- # {#safe_extract_path} is the final line of defense; this check just
475
- # fails fast with a clear message. Delegates to {Bulkrax::SafePath} so
476
- # cloud-file destinations and sources share the same rules.
477
- #
478
- # @raise [Bulkrax::UnzipError] if the entry name is unsafe
479
- <<<<<<< HEAD
480
- def reject_unsafe_entry!(name)
481
- return unless name.start_with?('/') || name.split('/').include?('..')
482
- raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: name)
483
- =======
484
- def self.reject_unsafe_entry!(name)
485
- Bulkrax::SafePath.reject_unsafe_entry!(name)
486
- rescue Bulkrax::UnsafePathError
487
- raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: name.dup.force_encoding(Encoding::UTF_8).scrub)
488
- >>>>>>> acc18e45 (Merge commit from fork)
489
- end
490
-
491
- # Zip Slip chokepoint. Resolves `relative_dest` against `dest_dir` and
492
- # returns the absolute destination path — but only if it stays inside
493
- # `dest_dir`, including through any symlink. Callers must use this
494
- # value rather than building their own path with `File.join`, so the
495
- # path returned is always safe by construction.
496
- #
497
- # @return [String] absolute destination path, validated to be inside `dest_dir`
498
- # @raise [Bulkrax::UnzipError] if `relative_dest` escapes `dest_dir`
499
- def safe_extract_path(dest_dir, relative_dest)
500
- Bulkrax::SafePath.confine!(dest_dir, relative_dest)
501
- rescue Bulkrax::UnsafePathError
502
- raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: relative_dest)
503
- end
504
-
505
- def copy_file(file_to_copy)
506
- destination = File.join(importer_unzip_path(mkdir: true), File.basename(file_to_copy))
507
- FileUtils.cp(file_to_copy, destination)
508
- end
509
-
510
- def untar(file_to_untar)
511
- Dir.mkdir(importer_unzip_path(mkdir: true)) unless File.directory?(importer_unzip_path(mkdir: true))
512
- command = "tar -xzf #{Shellwords.escape(file_to_untar)} -C #{Shellwords.escape(importer_unzip_path)}"
513
- result = system(command)
514
- raise "Failed to extract #{file_to_untar}" unless result
515
- end
516
-
517
- def zip
518
- FileUtils.mkdir_p(exporter_export_zip_path)
519
-
520
- Dir["#{exporter_export_path}/**"].each do |folder|
521
- zip_path = "#{exporter_export_zip_path.split('/').last}_#{folder.split('/').last}.zip"
522
- FileUtils.rm_rf("#{exporter_export_zip_path}/#{zip_path}")
523
-
524
- Zip::File.open(File.join("#{exporter_export_zip_path}/#{zip_path}"), create: true) do |zip_file|
525
- Dir["#{folder}/**/**"].each do |file|
526
- zip_file.add(file.sub("#{folder}/", ''), file)
527
- end
528
- end
529
- end
530
- end
531
-
532
- # Path for the import
533
- # @return [String]
534
- def import_file_path
535
- @import_file_path ||= real_import_file_path
536
- end
537
-
538
- private
539
-
540
- # @return [String]
541
- def real_import_file_path
542
- parser_fields['import_file_path']
543
- end
544
- end
545
- end
@@ -1,123 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Bulkrax
4
- # Validates a cloud-file source (the Hash a user selected via
5
- # browse-everything, or supplied directly in importer params) before
6
- # Bulkrax downloads it. Destination confinement (see {Bulkrax::SafePath})
7
- # stops a malicious filename from writing outside the importer's
8
- # workspace; this validator stops a malicious `url` from reading a local
9
- # file outside the configured filesystem-provider root, or from reaching
10
- # a private/internal network over HTTP(S) (SSRF).
11
- #
12
- # Called twice by design: once before enqueueing, so an obviously unsafe
13
- # batch never reaches a job queue, and again inside the download job,
14
- # so a legacy or tampered job argument can't bypass validation.
15
- class CloudSourceValidator
16
- LOCAL_FILE_HOSTS = ['', 'localhost'].freeze
17
-
18
- # Lightweight preflight for a single cloud-file selection: checks the
19
- # raw file name and the source's safety, without resolving anything
20
- # against an importer's workspace. Shared by CsvParser#plan_cloud_file,
21
- # BagitParser#retrieve_cloud_files, and ImportersController's pre-save
22
- # check, so a selection is judged unsafe the same way whether or not an
23
- # importer has been persisted yet.
24
- #
25
- # @param file [Hash] a single cloud-file spec with at least `url` and `file_name` keys
26
- # @raise [Bulkrax::UnsafePathError] if the raw file name is unsafe
27
- # @raise [Bulkrax::UnsafeCloudSourceError] if the source isn't safe to download
28
- # @return [true]
29
- def self.preflight!(file)
30
- name = file['file_name'] || file[:file_name]
31
- raise Bulkrax::UnsafePathError, 'cloud file name must not be blank' if name.blank?
32
- Bulkrax::SafePath.reject_unsafe_entry!(name)
33
- validate!(file)
34
- end
35
-
36
- # @param file [Hash] a single cloud-file spec with at least a `url` key
37
- # @raise [Bulkrax::UnsafeCloudSourceError] if the source isn't safe to download
38
- # @return [true]
39
- def self.validate!(file)
40
- url = file['url'] || file[:url]
41
- raise Bulkrax::UnsafeCloudSourceError, 'the cloud file is missing a url' if url.blank?
42
-
43
- uri = parse_uri(url)
44
- case uri.scheme
45
- when 'file'
46
- validate_file_scheme!(uri)
47
- when 'http', 'https'
48
- validate_http_scheme!(uri, file)
49
- else
50
- raise Bulkrax::UnsafeCloudSourceError, 'unsupported source scheme'
51
- end
52
- true
53
- end
54
-
55
- def self.parse_uri(url)
56
- Addressable::URI.parse(url.to_s)
57
- rescue Addressable::URI::InvalidURIError
58
- raise Bulkrax::UnsafeCloudSourceError, 'the cloud file url is malformed'
59
- end
60
- private_class_method :parse_uri
61
-
62
- def self.validate_file_scheme!(uri)
63
- raise Bulkrax::UnsafeCloudSourceError, 'a file: source must not specify a remote host' unless LOCAL_FILE_HOSTS.include?(uri.host.to_s)
64
-
65
- home = file_system_home
66
- raise Bulkrax::UnsafeCloudSourceError, 'the filesystem provider is not configured' if home.blank?
67
-
68
- decoded_path = URI::DEFAULT_PARSER.unescape(uri.path.to_s)
69
- Bulkrax::SafePath.confine!(home, decoded_path, require_file: true, allow_absolute: true)
70
- rescue Bulkrax::UnsafePathError => e
71
- raise Bulkrax::UnsafeCloudSourceError, e.message
72
- end
73
- private_class_method :validate_file_scheme!
74
-
75
- # Enforced at connection time, not by a DNS preflight: SsrfFilter
76
- # resolves the hostname, rejects private/loopback/link-local/metadata
77
- # addresses, and then connects directly to the validated IP (with TLS
78
- # verified against the original hostname), closing the gap a
79
- # resolve-then-connect check would leave open to DNS rebinding.
80
- # Redirects are disabled outright (rather than re-validated per hop)
81
- # so a header carrying provider credentials is never resent to a
82
- # different host.
83
- def self.validate_http_scheme!(uri, file)
84
- raise Bulkrax::UnsafeCloudSourceError, 'no remote cloud provider is configured' unless remote_provider_configured?
85
- raise Bulkrax::UnsafeCloudSourceError, 'the cloud file url must include a host' if uri.host.blank?
86
- raise Bulkrax::UnsafeCloudSourceError, 'the cloud file url must not include credentials' if uri.userinfo.present?
87
-
88
- SsrfFilter.head(uri.to_s, headers: request_headers(file), max_redirects: 0)
89
- rescue SsrfFilter::Error, SocketError, SystemCallError, Timeout::Error, OpenSSL::SSL::SSLError
90
- # Static and class-agnostic on purpose: SsrfFilter's own exception
91
- # message can include the resolved IP or hostname it rejected, and
92
- # this error is shown to the importing user / written to the
93
- # importer's status, not just an ops log.
94
- raise Bulkrax::UnsafeCloudSourceError, 'blocked unsafe remote source'
95
- end
96
- private_class_method :validate_http_scheme!
97
-
98
- # Merges the caller-supplied headers (used for provider auth) but
99
- # strips any transport-control header a malicious selection could use
100
- # to smuggle a different Host past SsrfFilter's own Host header.
101
- def self.request_headers(file)
102
- headers = {}
103
- headers.merge!(file['headers'] || file[:headers] || {})
104
- headers.merge!(file['auth_header'] || file[:auth_header] || {})
105
- headers.reject { |key, _| key.to_s.casecmp('host').zero? }
106
- end
107
-
108
- def self.file_system_home
109
- config = provider_config('file_system')
110
- config && (config['home'] || config[:home])
111
- end
112
-
113
- def self.remote_provider_configured?
114
- BrowseEverything.config.any? { |key, value| key.to_s != 'file_system' && value.present? }
115
- end
116
-
117
- def self.provider_config(name)
118
- config = BrowseEverything.config
119
- config[name] || config[name.to_sym]
120
- end
121
- private_class_method :provider_config
122
- end
123
- end