bulkrax 8.2.4 → 8.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. checksums.yaml +4 -4
  2. data/app/assets/javascripts/bulkrax/bulkrax.js +9 -7
  3. data/app/assets/javascripts/bulkrax/importers.js.erb +134 -99
  4. data/app/controllers/bulkrax/importers_controller.rb +28 -112
  5. data/app/controllers/concerns/bulkrax/datatables_behavior.rb +1 -1
  6. data/app/factories/bulkrax/object_factory.rb +1 -4
  7. data/app/jobs/bulkrax/download_cloud_file_job.rb +21 -156
  8. data/app/jobs/bulkrax/export_work_job.rb +1 -1
  9. data/app/models/bulkrax/status.rb +1 -1
  10. data/app/models/concerns/bulkrax/export_behavior.rb +4 -8
  11. data/app/parsers/bulkrax/application_parser.rb +2 -25
  12. data/app/parsers/bulkrax/bagit_parser.rb +3 -10
  13. data/app/parsers/bulkrax/csv_parser.rb +17 -51
  14. data/app/parsers/bulkrax/parser_export_record_set.rb +1 -1
  15. data/app/views/bulkrax/importers/_bagit_fields.html.erb +10 -3
  16. data/app/views/bulkrax/importers/_csv_fields.html.erb +13 -9
  17. data/app/views/bulkrax/importers/_file_uploader.html.erb +37 -0
  18. data/app/views/bulkrax/importers/_xml_fields.html.erb +10 -4
  19. data/config/locales/bulkrax.en.yml +0 -3
  20. data/lib/bulkrax/version.rb +1 -1
  21. data/lib/bulkrax.rb +0 -1
  22. data/lib/tasks/bulkrax_tasks.rake +1 -1
  23. metadata +9 -32
  24. data/app/errors/bulkrax/unsafe_cloud_source_error.rb +0 -11
  25. data/app/errors/bulkrax/unsafe_path_error.rb +0 -8
  26. data/app/parsers/bulkrax/application_parser.rb.orig +0 -545
  27. data/app/services/bulkrax/cloud_source_validator.rb +0 -123
  28. data/app/services/bulkrax/safe_path.rb +0 -142
  29. data/db/migrate/20241203010707_entry_error_denormalization.rb +0 -7
  30. data/db/migrate/20241205212513_faster_first_entry.rb +0 -7
metadata CHANGED
@@ -1,13 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bulkrax
3
3
  version: !ruby/object:Gem::Version
4
- version: 8.2.4
4
+ version: 8.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Rob Kaufman
8
+ autorequire:
8
9
  bindir: bin
9
10
  cert_chain: []
10
- date: 1980-01-02 00:00:00.000000000 Z
11
+ date: 2024-12-02 00:00:00.000000000 Z
11
12
  dependencies:
12
13
  - !ruby/object:Gem::Dependency
13
14
  name: rails
@@ -29,14 +30,14 @@ dependencies:
29
30
  requirements:
30
31
  - - "~>"
31
32
  - !ruby/object:Gem::Version
32
- version: 0.6.0
33
+ version: 0.4.6
33
34
  type: :runtime
34
35
  prerelease: false
35
36
  version_requirements: !ruby/object:Gem::Requirement
36
37
  requirements:
37
38
  - - "~>"
38
39
  - !ruby/object:Gem::Version
39
- version: 0.6.0
40
+ version: 0.4.6
40
41
  - !ruby/object:Gem::Dependency
41
42
  name: coderay
42
43
  requirement: !ruby/object:Gem::Requirement
@@ -237,26 +238,6 @@ dependencies:
237
238
  - - ">="
238
239
  - !ruby/object:Gem::Version
239
240
  version: '0'
240
- - !ruby/object:Gem::Dependency
241
- name: ssrf_filter
242
- requirement: !ruby/object:Gem::Requirement
243
- requirements:
244
- - - ">="
245
- - !ruby/object:Gem::Version
246
- version: '1.0'
247
- - - "<"
248
- - !ruby/object:Gem::Version
249
- version: '2.0'
250
- type: :runtime
251
- prerelease: false
252
- version_requirements: !ruby/object:Gem::Requirement
253
- requirements:
254
- - - ">="
255
- - !ruby/object:Gem::Version
256
- version: '1.0'
257
- - - "<"
258
- - !ruby/object:Gem::Version
259
- version: '2.0'
260
241
  - !ruby/object:Gem::Dependency
261
242
  name: dry-monads
262
243
  requirement: !ruby/object:Gem::Requirement
@@ -361,8 +342,6 @@ files:
361
342
  - app/controllers/concerns/bulkrax/api.rb
362
343
  - app/controllers/concerns/bulkrax/datatables_behavior.rb
363
344
  - app/controllers/concerns/bulkrax/download_behavior.rb
364
- - app/errors/bulkrax/unsafe_cloud_source_error.rb
365
- - app/errors/bulkrax/unsafe_path_error.rb
366
345
  - app/factories/bulkrax/object_factory.rb
367
346
  - app/factories/bulkrax/object_factory_interface.rb
368
347
  - app/factories/bulkrax/valkyrie_object_factory.rb
@@ -425,17 +404,14 @@ files:
425
404
  - app/models/concerns/bulkrax/importer_exporter_behavior.rb
426
405
  - app/models/concerns/bulkrax/status_info.rb
427
406
  - app/parsers/bulkrax/application_parser.rb
428
- - app/parsers/bulkrax/application_parser.rb.orig
429
407
  - app/parsers/bulkrax/bagit_parser.rb
430
408
  - app/parsers/bulkrax/csv_parser.rb
431
409
  - app/parsers/bulkrax/oai_dc_parser.rb
432
410
  - app/parsers/bulkrax/oai_qualified_dc_parser.rb
433
411
  - app/parsers/bulkrax/parser_export_record_set.rb
434
412
  - app/parsers/bulkrax/xml_parser.rb
435
- - app/services/bulkrax/cloud_source_validator.rb
436
413
  - app/services/bulkrax/factory_class_finder.rb
437
414
  - app/services/bulkrax/remove_relationships_for_importer.rb
438
- - app/services/bulkrax/safe_path.rb
439
415
  - app/services/hyrax/custom_queries/find_by_source_identifier.rb
440
416
  - app/services/wings/custom_queries/find_by_source_identifier.rb
441
417
  - app/views/bulkrax/entries/_parsed_metadata.html.erb
@@ -452,6 +428,7 @@ files:
452
428
  - app/views/bulkrax/importers/_csv_fields.html.erb
453
429
  - app/views/bulkrax/importers/_edit_form_buttons.html.erb
454
430
  - app/views/bulkrax/importers/_edit_item_buttons.html.erb
431
+ - app/views/bulkrax/importers/_file_uploader.html.erb
455
432
  - app/views/bulkrax/importers/_form.html.erb
456
433
  - app/views/bulkrax/importers/_oai_fields.html.erb
457
434
  - app/views/bulkrax/importers/_xml_fields.html.erb
@@ -510,8 +487,6 @@ files:
510
487
  - db/migrate/20240823173525_add_error_tracking_to_pending_relationships.rb
511
488
  - db/migrate/20240916182737_add_last_imported_at_to_bulkrax_importers.rb
512
489
  - db/migrate/20240916182823_add_next_import_at_to_bulkrax_importers.rb
513
- - db/migrate/20241203010707_entry_error_denormalization.rb
514
- - db/migrate/20241205212513_faster_first_entry.rb
515
490
  - lib/bulkrax.rb
516
491
  - lib/bulkrax/engine.rb
517
492
  - lib/bulkrax/entry_spec_helper.rb
@@ -529,6 +504,7 @@ homepage: https://github.com/samvera-labs/bulkrax
529
504
  licenses:
530
505
  - Apache-2.0
531
506
  metadata: {}
507
+ post_install_message:
532
508
  rdoc_options: []
533
509
  require_paths:
534
510
  - lib
@@ -543,7 +519,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
543
519
  - !ruby/object:Gem::Version
544
520
  version: '0'
545
521
  requirements: []
546
- rubygems_version: 3.6.8
522
+ rubygems_version: 3.4.20
523
+ signing_key:
547
524
  specification_version: 4
548
525
  summary: Import and export tool for Hyrax and Hyku
549
526
  test_files: []
@@ -1,11 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Bulkrax
4
- # Raised by {Bulkrax::CloudSourceValidator} when a cloud-file source
5
- # (the `url` a user selected via browse-everything, or supplied directly
6
- # in importer params) can't be safely downloaded: a scheme other than
7
- # `file`/`http`/`https`, a `file:` source outside the configured
8
- # filesystem-provider root, or an HTTP(S) source that resolves to a
9
- # private, loopback, or link-local/metadata address.
10
- class UnsafeCloudSourceError < StandardError; end
11
- end
@@ -1,8 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Bulkrax
4
- # Raised by {Bulkrax::SafePath} when a path would escape its configured
5
- # root, whether that root is a ZIP extraction directory, an importer's
6
- # trusted workspace, or a configured filesystem-provider home.
7
- class UnsafePathError < StandardError; end
8
- end
@@ -1,545 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Bulkrax
4
- # An abstract class that establishes the API for Bulkrax's import and export parsing.
5
- #
6
- # @abstract Subclass the Bulkrax::ApplicationParser to create a parser that handles a specific format (e.g. CSV, Bagit, XML, etc).
7
- class ApplicationParser # rubocop:disable Metrics/ClassLength
8
- attr_accessor :importerexporter, :headers
9
- alias importer importerexporter
10
- alias exporter importerexporter
11
- delegate :only_updates, :limit, :current_run, :errors, :mapping,
12
- :seen, :increment_counters, :parser_fields, :user, :keys_without_numbers,
13
- :key_without_numbers, :status, :set_status_info, :status_info, :status_at,
14
- :exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
15
- :zip?, :file?, :remove_and_rerun, :zip_file?,
16
- to: :importerexporter
17
-
18
- # @todo Convert to `class_attribute :parser_fiels, default: {}`
19
- def self.parser_fields
20
- {}
21
- end
22
-
23
- # @return [TrueClass,FalseClass] this parser does or does not support exports.
24
- #
25
- # @todo Convert to `class_attribute :export_supported, default: false, instance_predicate: true` and `self << class; alias export_supported? export_supported; end`
26
- def self.export_supported?
27
- false
28
- end
29
-
30
- # @return [TrueClass,FalseClass] this parser does or does not support imports.
31
- #
32
- # @todo Convert to `class_attribute :import_supported, default: false, instance_predicate: true` and `self << class; alias import_supported? import_supported; end`
33
- def self.import_supported?
34
- true
35
- end
36
-
37
- def initialize(importerexporter)
38
- @importerexporter = importerexporter
39
- @headers = []
40
- end
41
-
42
- # @api public
43
- # @abstract Subclass and override {#entry_class} to implement behavior for the parser.
44
- def entry_class
45
- raise NotImplementedError, 'must be defined'
46
- end
47
-
48
- def work_entry_class
49
- entry_class
50
- end
51
-
52
- # @api public
53
- # @abstract Subclass and override {#collection_entry_class} to implement behavior for the parser.
54
- def collection_entry_class
55
- raise NotImplementedError, 'must be defined'
56
- end
57
-
58
- # @api public
59
- # @abstract Subclass and override {#file_set_entry_class} to implement behavior for the parser.
60
- def file_set_entry_class
61
- raise NotImplementedError, 'must be defined'
62
- end
63
-
64
- # @api public
65
- # @abstract Subclass and override {#records} to implement behavior for the parser.
66
- def records(_opts = {})
67
- raise NotImplementedError, 'must be defined'
68
- end
69
-
70
- # @return [Symbol] the name of the identifying property in the source system from which we're
71
- # importing (e.g. is *not* this application that mounts *this* Bulkrax engine).
72
- #
73
- # @see #work_identifier
74
- # @see https://github.com/samvera-labs/bulkrax/wiki/CSV-Importer#source-identifier Bulkrax Wiki regarding source identifier
75
- def source_identifier
76
- @source_identifier ||= get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('from')&.first&.to_sym || :source_identifier
77
- end
78
-
79
- # @return [Symbol] the name of the identifying property for the system which we're importing
80
- # into (e.g. the application that mounts *this* Bulkrax engine)
81
- # @see #source_identifier
82
- def work_identifier
83
- @work_identifier ||= get_field_mapping_hash_for('source_identifier')&.keys&.first&.to_sym || :source
84
- end
85
-
86
- # @return [Symbol] the solr property of the source_identifier. Used for searching.
87
- # defaults to work_identifier value + "_sim"
88
- # @see #work_identifier
89
- def work_identifier_search_field
90
- @work_identifier_search_field ||= Array.wrap(get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('search_field'))&.first&.to_s || "#{work_identifier}_sim"
91
- end
92
-
93
- # @return [String]
94
- def generated_metadata_mapping
95
- @generated_metadata_mapping ||= 'generated'
96
- end
97
-
98
- # @return [String, NilClass]
99
- # @see #related_parents_raw_mapping
100
- def related_parents_raw_mapping
101
- @related_parents_raw_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.values&.first&.[]('from')&.first
102
- end
103
-
104
- # @return [String]
105
- # @see #related_parents_field_mapping
106
- def related_parents_parsed_mapping
107
- @related_parents_parsed_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.keys&.first || 'parents'
108
- end
109
-
110
- # @return [String, NilClass]
111
- # @see #related_children_parsed_mapping
112
- def related_children_raw_mapping
113
- @related_children_raw_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.values&.first&.[]('from')&.first
114
- end
115
-
116
- # @return [String]
117
- # @see #related_children_raw_mapping
118
- def related_children_parsed_mapping
119
- @related_children_parsed_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.keys&.first || 'children'
120
- end
121
-
122
- # @api private
123
- def get_field_mapping_hash_for(key)
124
- return instance_variable_get("@#{key}_hash") if instance_variable_get("@#{key}_hash").present?
125
-
126
- mapping = importerexporter.field_mapping.is_a?(Hash) ? importerexporter.field_mapping : {}
127
- instance_variable_set(
128
- "@#{key}_hash",
129
- mapping&.with_indifferent_access&.select { |_, h| h.key?(key) }
130
- )
131
- raise StandardError, "more than one #{key} declared: #{instance_variable_get("@#{key}_hash").keys.join(', ')}" if instance_variable_get("@#{key}_hash").length > 1
132
-
133
- instance_variable_get("@#{key}_hash")
134
- end
135
-
136
- # @return [Array<String>]
137
- def model_field_mappings
138
- model_mappings = Bulkrax.field_mappings[self.class.to_s]&.dig('model', :from) || []
139
- model_mappings |= ['model']
140
-
141
- model_mappings
142
- end
143
-
144
- # @return [String]
145
- def perform_method
146
- if self.validate_only
147
- 'perform_now'
148
- else
149
- 'perform_later'
150
- end
151
- end
152
-
153
- # The visibility of the record. Acceptable values are: "open", "embargo", "lease", "authenticated", "restricted". The default is "open"
154
- #
155
- # @return [String]
156
- # @see https://github.com/samvera/hydra-head/blob/main/hydra-access-controls/app/models/concerns/hydra/access_controls/access_right.rb Hydra::AccessControls::AccessRight for details on the range of values.
157
- # @see https://github.com/samvera/hyrax/blob/bd2bcffc33e183904be2c175367648815f25bc2b/app/services/hyrax/visibility_intention.rb Hyrax::VisibilityIntention for how we process the visibility.
158
- def visibility
159
- @visibility ||= self.parser_fields['visibility'] || 'open'
160
- end
161
-
162
- def create_collections
163
- create_objects(['collection'])
164
- end
165
-
166
- def create_works
167
- create_objects(['work'])
168
- end
169
-
170
- def create_file_sets
171
- create_objects(['file_set'])
172
- end
173
-
174
- def create_relationships
175
- create_objects(['relationship'])
176
- end
177
-
178
- # @api public
179
- #
180
- # @param types [Array<Symbol>] the types of objects that we'll create.
181
- #
182
- # @see Bulkrax::Importer::DEFAULT_OBJECT_TYPES
183
- # @see #create_collections
184
- # @see #create_works
185
- # @see #create_file_sets
186
- # @see #create_relationships
187
- def create_objects(types_array = nil)
188
- index = 0
189
- (types_array || %w[collection work file_set relationship]).each do |type|
190
- if type.eql?('relationship')
191
- ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: importerexporter.id)
192
- next
193
- end
194
- send(type.pluralize).each do |current_record|
195
- next unless record_has_source_identifier(current_record, index)
196
- break if limit_reached?(limit, index)
197
- seen[current_record[source_identifier]] = true
198
- create_entry_and_job(current_record, type)
199
- increment_counters(index, "#{type}": true)
200
- index += 1
201
- end
202
- importer.record_status
203
- end
204
- true
205
- rescue StandardError => e
206
- set_status_info(e)
207
- end
208
-
209
- def rebuild_entries(types_array = nil)
210
- index = 0
211
- (types_array || %w[collection work file_set relationship]).each do |type|
212
- # works are not guaranteed to have Work in the type
213
- if type.eql?('relationship')
214
- ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: importerexporter.id)
215
- next
216
- end
217
- importer.entries.where(rebuild_entry_query(type, parser_fields['entry_statuses'])).find_each do |e|
218
- seen[e.identifier] = true
219
- e.status_info('Pending', importer.current_run)
220
- if remove_and_rerun
221
- delay = calculate_type_delay(type)
222
- "Bulkrax::DeleteAndImport#{type.camelize}Job".constantize.set(wait: delay).send(perform_method, e, current_run)
223
- else
224
- "Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, e.id, current_run.id)
225
- end
226
- increment_counters(index)
227
- index += 1
228
- end
229
- end
230
- end
231
-
232
- def rebuild_entry_query(type, statuses)
233
- type_col = Bulkrax::Entry.arel_table['type']
234
- status_col = Bulkrax::Entry.arel_table['status_message']
235
-
236
- query = (type == 'work' ? type_col.does_not_match_all(%w[collection file_set]) : type_col.matches(type.camelize))
237
- query.and(status_col.in(statuses))
238
- end
239
-
240
- def calculate_type_delay(type)
241
- return 2.minutes if type == 'file_set'
242
- return 1.minute if type == 'work'
243
- return 0
244
- end
245
-
246
- def record_raw_metadata(record)
247
- record.to_h
248
- end
249
-
250
- def record_deleted?(record)
251
- return false unless record.key?(:delete)
252
- ActiveModel::Type::Boolean.new.cast(record[:delete])
253
- end
254
-
255
- def record_remove_and_rerun?(record)
256
- return false unless record.key?(:remove_and_rerun)
257
- ActiveModel::Type::Boolean.new.cast(record[:remove_and_rerun])
258
- end
259
-
260
- def create_entry_and_job(current_record, type, identifier = nil)
261
- identifier ||= current_record[source_identifier]
262
- new_entry = find_or_create_entry(send("#{type}_entry_class"),
263
- identifier,
264
- 'Bulkrax::Importer',
265
- record_raw_metadata(current_record))
266
- new_entry.status_info('Pending', importer.current_run)
267
- if record_deleted?(current_record)
268
- "Bulkrax::Delete#{type.camelize}Job".constantize.send(perform_method, new_entry, current_run)
269
- elsif record_remove_and_rerun?(current_record) || remove_and_rerun
270
- delay = calculate_type_delay(type)
271
- "Bulkrax::DeleteAndImport#{type.camelize}Job".constantize.set(wait: delay).send(perform_method, new_entry, current_run)
272
- else
273
- "Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, new_entry.id, current_run.id)
274
- end
275
- end
276
-
277
- # Optional, define if using browse everything for file upload
278
- def retrieve_cloud_files(_files, _importer); end
279
-
280
- # @param file [#path, #original_filename] the file object that with the relevant data for the
281
- # import.
282
- def write_import_file(file)
283
- path = File.join(path_for_import, file.original_filename)
284
- FileUtils.mv(
285
- file.path,
286
- path
287
- )
288
- path
289
- end
290
-
291
- # Base path for imported and exported files
292
- # @param [String]
293
- # @return [String] the base path for files that this parser will "parse"
294
- def base_path(type = 'import')
295
- # account for multiple versions of hyku
296
- is_multitenant = ENV['HYKU_MULTITENANT'] == 'true' || ENV['SETTINGS__MULTITENANCY__ENABLED'] == 'true'
297
- is_multitenant ? File.join(Bulkrax.send("#{type}_path"), ::Site.instance.account.name) : Bulkrax.send("#{type}_path")
298
- end
299
-
300
- # Path where we'll store the import metadata and files
301
- # this is used for uploaded and cloud files
302
- # @return [String]
303
- def path_for_import
304
- @path_for_import = File.join(base_path, importerexporter.path_string)
305
- FileUtils.mkdir_p(@path_for_import) unless File.exist?(@path_for_import)
306
- @path_for_import
307
- end
308
-
309
- # @abstract Subclass and override {#setup_export_file} to implement behavior for the parser.
310
- def setup_export_file
311
- raise NotImplementedError, 'must be defined' if exporter?
312
- end
313
-
314
- # @abstract Subclass and override {#write_files} to implement behavior for the parser.
315
- def write_files
316
- raise NotImplementedError, 'must be defined' if exporter?
317
- end
318
-
319
- # @return [TrueClass,FalseClass]
320
- def importer?
321
- importerexporter.is_a?(Bulkrax::Importer)
322
- end
323
-
324
- # @return [TrueClass,FalseClass]
325
- def exporter?
326
- importerexporter.is_a?(Bulkrax::Exporter)
327
- end
328
-
329
- # @param limit [Integer] limit set on the importerexporter
330
- # @param index [Integer] index of current iteration
331
- # @return [TrueClass,FalseClass]
332
- def limit_reached?(limit, index)
333
- return false if limit.nil? || limit.zero? # no limit
334
- index >= limit
335
- end
336
-
337
- # Override to add specific validations
338
- # @return [TrueClass,FalseClass]
339
- def valid_import?
340
- true
341
- end
342
-
343
- # @return [TrueClass,FalseClass]
344
- def record_has_source_identifier(record, index)
345
- if record[source_identifier].blank?
346
- if Bulkrax.fill_in_blank_source_identifiers.present?
347
- record[source_identifier] = Bulkrax.fill_in_blank_source_identifiers.call(self, index)
348
- else
349
- invalid_record("Missing #{source_identifier} for #{record.to_h}\n")
350
- false
351
- end
352
- else
353
- true
354
- end
355
- end
356
-
357
- # rubocop:disable Rails/SkipsModelValidations
358
- def invalid_record(message)
359
- current_run.invalid_records ||= ""
360
- current_run.invalid_records += message
361
- current_run.save
362
- ImporterRun.increment_counter(:failed_records, current_run.id)
363
- ImporterRun.decrement_counter(:enqueued_records, current_run.id) unless ImporterRun.find(current_run.id).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
364
- end
365
- # rubocop:enable Rails/SkipsModelValidations
366
-
367
- # @return [Array<String>]
368
- def required_elements
369
- matched_elements = ((importerexporter.mapping.keys || []) & (Bulkrax.required_elements || []))
370
- unless matched_elements.count == Bulkrax.required_elements.count
371
- missing_elements = Bulkrax.required_elements - matched_elements
372
- error_alert = "Missing mapping for at least one required element, missing mappings are: #{missing_elements.join(', ')}"
373
- raise StandardError, error_alert
374
- end
375
- if Bulkrax.fill_in_blank_source_identifiers
376
- Bulkrax.required_elements
377
- else
378
- Bulkrax.required_elements + [source_identifier]
379
- end
380
- end
381
-
382
- def new_entry(entryclass, type)
383
- entryclass.new(
384
- importerexporter_id: importerexporter.id,
385
- importerexporter_type: type
386
- )
387
- end
388
-
389
- def find_or_create_entry(entryclass, identifier, type, raw_metadata = nil)
390
- # limit entry search to just this importer or exporter. Don't go moving them
391
- entry = importerexporter.entries.where(
392
- identifier: identifier
393
- ).first
394
- entry ||= entryclass.new(
395
- importerexporter_id: importerexporter.id,
396
- importerexporter_type: type,
397
- identifier: identifier
398
- )
399
- entry.raw_metadata = raw_metadata
400
- # Setting parsed_metadata specifically for the id so we can find the object via the
401
- # id in a delete. This is likely to get clobbered in a regular import, which is fine.
402
- entry.parsed_metadata = { id: raw_metadata['id'] } if raw_metadata&.key?('id')
403
- entry.save!
404
- entry
405
- end
406
-
407
- # @todo - review this method - is it ever used?
408
- def record(identifier, _opts = {})
409
- return @record if @record
410
-
411
- @record = entry_class.new(self, identifier)
412
- @record.build
413
- return @record
414
- end
415
-
416
- def total
417
- 0
418
- end
419
-
420
- def collections_total
421
- 0
422
- end
423
-
424
- def file_sets_total
425
- 0
426
- end
427
-
428
- def write
429
- write_files
430
- zip
431
- end
432
-
433
- # Extracts a zip verbatim into {#importer_unzip_path}, preserving the zip's
434
- # internal structure. Filters macOS junk (`__MACOSX/`, `.DS_Store`, `._*`).
435
- # Parser subclasses that need to interpret the zip's structure (e.g.
436
- # {Bulkrax::CsvParser#unzip_with_primary_csv}) should call a more specific
437
- # method rather than this one.
438
- def unzip(file_to_unzip)
439
- return untar(file_to_unzip) if file_to_unzip.end_with?('.tar.gz')
440
-
441
- dest_dir = importer_unzip_path(mkdir: true)
442
- Zip::File.open(file_to_unzip) do |zip_file|
443
- zip_file.each do |entry|
444
- next unless entry.file?
445
- next if macos_junk_entry?(entry.name)
446
- reject_unsafe_entry!(entry.name)
447
- dest_path = safe_extract_path(dest_dir, entry.name)
448
- FileUtils.mkdir_p(File.dirname(dest_path))
449
- next if File.exist?(dest_path)
450
- extract_zip_entry(zip_file, entry, dest_dir, entry.name, dest_path)
451
- end
452
- end
453
- end
454
-
455
- # rubyzip 2.x: extract(entry, absolute_dest_path)
456
- # rubyzip 3.x: extract(entry, relative_name, destination_directory: dir)
457
- #
458
- # Callers are responsible for passing a `dest_path` produced by
459
- # {#safe_extract_path} so the write can't escape `dest_dir`.
460
- def extract_zip_entry(zip_file, entry, dest_dir, relative_name, dest_path)
461
- if zip_file.method(:extract).arity == 2
462
- zip_file.extract(entry, dest_path)
463
- else
464
- zip_file.extract(entry, relative_name, destination_directory: dest_dir)
465
- end
466
- end
467
-
468
- def macos_junk_entry?(name)
469
- name.start_with?('__MACOSX/') || name.split('/').any? { |part| part == '.DS_Store' || part.start_with?('._') }
470
- end
471
-
472
- # Zip Slip preflight — reject entries whose names are obviously unsafe
473
- # (absolute paths, `..` segments) before we touch the filesystem.
474
- # {#safe_extract_path} is the final line of defense; this check just
475
- # fails fast with a clear message. Delegates to {Bulkrax::SafePath} so
476
- # cloud-file destinations and sources share the same rules.
477
- #
478
- # @raise [Bulkrax::UnzipError] if the entry name is unsafe
479
- <<<<<<< HEAD
480
- def reject_unsafe_entry!(name)
481
- return unless name.start_with?('/') || name.split('/').include?('..')
482
- raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: name)
483
- =======
484
- def self.reject_unsafe_entry!(name)
485
- Bulkrax::SafePath.reject_unsafe_entry!(name)
486
- rescue Bulkrax::UnsafePathError
487
- raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: name.dup.force_encoding(Encoding::UTF_8).scrub)
488
- >>>>>>> acc18e45 (Merge commit from fork)
489
- end
490
-
491
- # Zip Slip chokepoint. Resolves `relative_dest` against `dest_dir` and
492
- # returns the absolute destination path — but only if it stays inside
493
- # `dest_dir`, including through any symlink. Callers must use this
494
- # value rather than building their own path with `File.join`, so the
495
- # path returned is always safe by construction.
496
- #
497
- # @return [String] absolute destination path, validated to be inside `dest_dir`
498
- # @raise [Bulkrax::UnzipError] if `relative_dest` escapes `dest_dir`
499
- def safe_extract_path(dest_dir, relative_dest)
500
- Bulkrax::SafePath.confine!(dest_dir, relative_dest)
501
- rescue Bulkrax::UnsafePathError
502
- raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: relative_dest)
503
- end
504
-
505
- def copy_file(file_to_copy)
506
- destination = File.join(importer_unzip_path(mkdir: true), File.basename(file_to_copy))
507
- FileUtils.cp(file_to_copy, destination)
508
- end
509
-
510
- def untar(file_to_untar)
511
- Dir.mkdir(importer_unzip_path(mkdir: true)) unless File.directory?(importer_unzip_path(mkdir: true))
512
- command = "tar -xzf #{Shellwords.escape(file_to_untar)} -C #{Shellwords.escape(importer_unzip_path)}"
513
- result = system(command)
514
- raise "Failed to extract #{file_to_untar}" unless result
515
- end
516
-
517
- def zip
518
- FileUtils.mkdir_p(exporter_export_zip_path)
519
-
520
- Dir["#{exporter_export_path}/**"].each do |folder|
521
- zip_path = "#{exporter_export_zip_path.split('/').last}_#{folder.split('/').last}.zip"
522
- FileUtils.rm_rf("#{exporter_export_zip_path}/#{zip_path}")
523
-
524
- Zip::File.open(File.join("#{exporter_export_zip_path}/#{zip_path}"), create: true) do |zip_file|
525
- Dir["#{folder}/**/**"].each do |file|
526
- zip_file.add(file.sub("#{folder}/", ''), file)
527
- end
528
- end
529
- end
530
- end
531
-
532
- # Path for the import
533
- # @return [String]
534
- def import_file_path
535
- @import_file_path ||= real_import_file_path
536
- end
537
-
538
- private
539
-
540
- # @return [String]
541
- def real_import_file_path
542
- parser_fields['import_file_path']
543
- end
544
- end
545
- end