bulkrax 9.5.1 → 9.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/controllers/bulkrax/importers_controller.rb +58 -17
- data/app/controllers/concerns/bulkrax/importer_file_handler.rb +54 -8
- data/app/errors/bulkrax/unsafe_cloud_source_error.rb +11 -0
- data/app/errors/bulkrax/unsafe_path_error.rb +8 -0
- data/app/jobs/bulkrax/download_cloud_file_job.rb +156 -21
- data/app/parsers/bulkrax/application_parser.rb +10 -10
- data/app/parsers/bulkrax/application_parser.rb.orig +545 -0
- data/app/parsers/bulkrax/bagit_parser.rb +10 -3
- data/app/parsers/bulkrax/csv_parser.rb +46 -15
- data/app/services/bulkrax/cloud_source_validator.rb +123 -0
- data/app/services/bulkrax/safe_path.rb +129 -0
- data/config/locales/bulkrax.en.yml +2 -0
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +1 -0
- metadata +28 -7
- data/app/factories/bulkrax/valkyrize-hyku.code-workspace +0 -19
|
@@ -0,0 +1,545 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
# An abstract class that establishes the API for Bulkrax's import and export parsing.
|
|
5
|
+
#
|
|
6
|
+
# @abstract Subclass the Bulkrax::ApplicationParser to create a parser that handles a specific format (e.g. CSV, Bagit, XML, etc).
|
|
7
|
+
class ApplicationParser # rubocop:disable Metrics/ClassLength
|
|
8
|
+
attr_accessor :importerexporter, :headers
|
|
9
|
+
alias importer importerexporter
|
|
10
|
+
alias exporter importerexporter
|
|
11
|
+
delegate :only_updates, :limit, :current_run, :errors, :mapping,
|
|
12
|
+
:seen, :increment_counters, :parser_fields, :user, :keys_without_numbers,
|
|
13
|
+
:key_without_numbers, :status, :set_status_info, :status_info, :status_at,
|
|
14
|
+
:exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
|
|
15
|
+
:zip?, :file?, :remove_and_rerun, :zip_file?,
|
|
16
|
+
to: :importerexporter
|
|
17
|
+
|
|
18
|
+
# @todo Convert to `class_attribute :parser_fiels, default: {}`
|
|
19
|
+
def self.parser_fields
|
|
20
|
+
{}
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# @return [TrueClass,FalseClass] this parser does or does not support exports.
|
|
24
|
+
#
|
|
25
|
+
# @todo Convert to `class_attribute :export_supported, default: false, instance_predicate: true` and `self << class; alias export_supported? export_supported; end`
|
|
26
|
+
def self.export_supported?
|
|
27
|
+
false
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# @return [TrueClass,FalseClass] this parser does or does not support imports.
|
|
31
|
+
#
|
|
32
|
+
# @todo Convert to `class_attribute :import_supported, default: false, instance_predicate: true` and `self << class; alias import_supported? import_supported; end`
|
|
33
|
+
def self.import_supported?
|
|
34
|
+
true
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def initialize(importerexporter)
|
|
38
|
+
@importerexporter = importerexporter
|
|
39
|
+
@headers = []
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# @api public
|
|
43
|
+
# @abstract Subclass and override {#entry_class} to implement behavior for the parser.
|
|
44
|
+
def entry_class
|
|
45
|
+
raise NotImplementedError, 'must be defined'
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def work_entry_class
|
|
49
|
+
entry_class
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# @api public
|
|
53
|
+
# @abstract Subclass and override {#collection_entry_class} to implement behavior for the parser.
|
|
54
|
+
def collection_entry_class
|
|
55
|
+
raise NotImplementedError, 'must be defined'
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# @api public
|
|
59
|
+
# @abstract Subclass and override {#file_set_entry_class} to implement behavior for the parser.
|
|
60
|
+
def file_set_entry_class
|
|
61
|
+
raise NotImplementedError, 'must be defined'
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# @api public
|
|
65
|
+
# @abstract Subclass and override {#records} to implement behavior for the parser.
|
|
66
|
+
def records(_opts = {})
|
|
67
|
+
raise NotImplementedError, 'must be defined'
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# @return [Symbol] the name of the identifying property in the source system from which we're
|
|
71
|
+
# importing (e.g. is *not* this application that mounts *this* Bulkrax engine).
|
|
72
|
+
#
|
|
73
|
+
# @see #work_identifier
|
|
74
|
+
# @see https://github.com/samvera-labs/bulkrax/wiki/CSV-Importer#source-identifier Bulkrax Wiki regarding source identifier
|
|
75
|
+
def source_identifier
|
|
76
|
+
@source_identifier ||= get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('from')&.first&.to_sym || :source_identifier
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# @return [Symbol] the name of the identifying property for the system which we're importing
|
|
80
|
+
# into (e.g. the application that mounts *this* Bulkrax engine)
|
|
81
|
+
# @see #source_identifier
|
|
82
|
+
def work_identifier
|
|
83
|
+
@work_identifier ||= get_field_mapping_hash_for('source_identifier')&.keys&.first&.to_sym || :source
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# @return [Symbol] the solr property of the source_identifier. Used for searching.
|
|
87
|
+
# defaults to work_identifier value + "_sim"
|
|
88
|
+
# @see #work_identifier
|
|
89
|
+
def work_identifier_search_field
|
|
90
|
+
@work_identifier_search_field ||= Array.wrap(get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('search_field'))&.first&.to_s || "#{work_identifier}_sim"
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# @return [String]
|
|
94
|
+
def generated_metadata_mapping
|
|
95
|
+
@generated_metadata_mapping ||= 'generated'
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# @return [String, NilClass]
|
|
99
|
+
# @see #related_parents_raw_mapping
|
|
100
|
+
def related_parents_raw_mapping
|
|
101
|
+
@related_parents_raw_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.values&.first&.[]('from')&.first
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# @return [String]
|
|
105
|
+
# @see #related_parents_field_mapping
|
|
106
|
+
def related_parents_parsed_mapping
|
|
107
|
+
@related_parents_parsed_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.keys&.first || 'parents'
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# @return [String, NilClass]
|
|
111
|
+
# @see #related_children_parsed_mapping
|
|
112
|
+
def related_children_raw_mapping
|
|
113
|
+
@related_children_raw_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.values&.first&.[]('from')&.first
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# @return [String]
|
|
117
|
+
# @see #related_children_raw_mapping
|
|
118
|
+
def related_children_parsed_mapping
|
|
119
|
+
@related_children_parsed_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.keys&.first || 'children'
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# @api private
|
|
123
|
+
def get_field_mapping_hash_for(key)
|
|
124
|
+
return instance_variable_get("@#{key}_hash") if instance_variable_get("@#{key}_hash").present?
|
|
125
|
+
|
|
126
|
+
mapping = importerexporter.field_mapping.is_a?(Hash) ? importerexporter.field_mapping : {}
|
|
127
|
+
instance_variable_set(
|
|
128
|
+
"@#{key}_hash",
|
|
129
|
+
mapping&.with_indifferent_access&.select { |_, h| h.key?(key) }
|
|
130
|
+
)
|
|
131
|
+
raise StandardError, "more than one #{key} declared: #{instance_variable_get("@#{key}_hash").keys.join(', ')}" if instance_variable_get("@#{key}_hash").length > 1
|
|
132
|
+
|
|
133
|
+
instance_variable_get("@#{key}_hash")
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# @return [Array<String>]
|
|
137
|
+
def model_field_mappings
|
|
138
|
+
model_mappings = Bulkrax.field_mappings[self.class.to_s]&.dig('model', :from) || []
|
|
139
|
+
model_mappings |= ['model']
|
|
140
|
+
|
|
141
|
+
model_mappings
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# @return [String]
|
|
145
|
+
def perform_method
|
|
146
|
+
if self.validate_only
|
|
147
|
+
'perform_now'
|
|
148
|
+
else
|
|
149
|
+
'perform_later'
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# The visibility of the record. Acceptable values are: "open", "embargo", "lease", "authenticated", "restricted". The default is "open"
|
|
154
|
+
#
|
|
155
|
+
# @return [String]
|
|
156
|
+
# @see https://github.com/samvera/hydra-head/blob/main/hydra-access-controls/app/models/concerns/hydra/access_controls/access_right.rb Hydra::AccessControls::AccessRight for details on the range of values.
|
|
157
|
+
# @see https://github.com/samvera/hyrax/blob/bd2bcffc33e183904be2c175367648815f25bc2b/app/services/hyrax/visibility_intention.rb Hyrax::VisibilityIntention for how we process the visibility.
|
|
158
|
+
def visibility
|
|
159
|
+
@visibility ||= self.parser_fields['visibility'] || 'open'
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def create_collections
|
|
163
|
+
create_objects(['collection'])
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def create_works
|
|
167
|
+
create_objects(['work'])
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def create_file_sets
|
|
171
|
+
create_objects(['file_set'])
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def create_relationships
|
|
175
|
+
create_objects(['relationship'])
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# @api public
|
|
179
|
+
#
|
|
180
|
+
# @param types [Array<Symbol>] the types of objects that we'll create.
|
|
181
|
+
#
|
|
182
|
+
# @see Bulkrax::Importer::DEFAULT_OBJECT_TYPES
|
|
183
|
+
# @see #create_collections
|
|
184
|
+
# @see #create_works
|
|
185
|
+
# @see #create_file_sets
|
|
186
|
+
# @see #create_relationships
|
|
187
|
+
def create_objects(types_array = nil)
|
|
188
|
+
index = 0
|
|
189
|
+
(types_array || %w[collection work file_set relationship]).each do |type|
|
|
190
|
+
if type.eql?('relationship')
|
|
191
|
+
ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: importerexporter.id)
|
|
192
|
+
next
|
|
193
|
+
end
|
|
194
|
+
send(type.pluralize).each do |current_record|
|
|
195
|
+
next unless record_has_source_identifier(current_record, index)
|
|
196
|
+
break if limit_reached?(limit, index)
|
|
197
|
+
seen[current_record[source_identifier]] = true
|
|
198
|
+
create_entry_and_job(current_record, type)
|
|
199
|
+
increment_counters(index, "#{type}": true)
|
|
200
|
+
index += 1
|
|
201
|
+
end
|
|
202
|
+
importer.record_status
|
|
203
|
+
end
|
|
204
|
+
true
|
|
205
|
+
rescue StandardError => e
|
|
206
|
+
set_status_info(e)
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def rebuild_entries(types_array = nil)
|
|
210
|
+
index = 0
|
|
211
|
+
(types_array || %w[collection work file_set relationship]).each do |type|
|
|
212
|
+
# works are not guaranteed to have Work in the type
|
|
213
|
+
if type.eql?('relationship')
|
|
214
|
+
ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: importerexporter.id)
|
|
215
|
+
next
|
|
216
|
+
end
|
|
217
|
+
importer.entries.where(rebuild_entry_query(type, parser_fields['entry_statuses'])).find_each do |e|
|
|
218
|
+
seen[e.identifier] = true
|
|
219
|
+
e.status_info('Pending', importer.current_run)
|
|
220
|
+
if remove_and_rerun
|
|
221
|
+
delay = calculate_type_delay(type)
|
|
222
|
+
"Bulkrax::DeleteAndImport#{type.camelize}Job".constantize.set(wait: delay).send(perform_method, e, current_run)
|
|
223
|
+
else
|
|
224
|
+
"Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, e.id, current_run.id)
|
|
225
|
+
end
|
|
226
|
+
increment_counters(index)
|
|
227
|
+
index += 1
|
|
228
|
+
end
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
def rebuild_entry_query(type, statuses)
|
|
233
|
+
type_col = Bulkrax::Entry.arel_table['type']
|
|
234
|
+
status_col = Bulkrax::Entry.arel_table['status_message']
|
|
235
|
+
|
|
236
|
+
query = (type == 'work' ? type_col.does_not_match_all(%w[collection file_set]) : type_col.matches(type.camelize))
|
|
237
|
+
query.and(status_col.in(statuses))
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def calculate_type_delay(type)
|
|
241
|
+
return 2.minutes if type == 'file_set'
|
|
242
|
+
return 1.minute if type == 'work'
|
|
243
|
+
return 0
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def record_raw_metadata(record)
|
|
247
|
+
record.to_h
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
def record_deleted?(record)
|
|
251
|
+
return false unless record.key?(:delete)
|
|
252
|
+
ActiveModel::Type::Boolean.new.cast(record[:delete])
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
def record_remove_and_rerun?(record)
|
|
256
|
+
return false unless record.key?(:remove_and_rerun)
|
|
257
|
+
ActiveModel::Type::Boolean.new.cast(record[:remove_and_rerun])
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
def create_entry_and_job(current_record, type, identifier = nil)
|
|
261
|
+
identifier ||= current_record[source_identifier]
|
|
262
|
+
new_entry = find_or_create_entry(send("#{type}_entry_class"),
|
|
263
|
+
identifier,
|
|
264
|
+
'Bulkrax::Importer',
|
|
265
|
+
record_raw_metadata(current_record))
|
|
266
|
+
new_entry.status_info('Pending', importer.current_run)
|
|
267
|
+
if record_deleted?(current_record)
|
|
268
|
+
"Bulkrax::Delete#{type.camelize}Job".constantize.send(perform_method, new_entry, current_run)
|
|
269
|
+
elsif record_remove_and_rerun?(current_record) || remove_and_rerun
|
|
270
|
+
delay = calculate_type_delay(type)
|
|
271
|
+
"Bulkrax::DeleteAndImport#{type.camelize}Job".constantize.set(wait: delay).send(perform_method, new_entry, current_run)
|
|
272
|
+
else
|
|
273
|
+
"Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, new_entry.id, current_run.id)
|
|
274
|
+
end
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
# Optional, define if using browse everything for file upload
|
|
278
|
+
def retrieve_cloud_files(_files, _importer); end
|
|
279
|
+
|
|
280
|
+
# @param file [#path, #original_filename] the file object that with the relevant data for the
|
|
281
|
+
# import.
|
|
282
|
+
def write_import_file(file)
|
|
283
|
+
path = File.join(path_for_import, file.original_filename)
|
|
284
|
+
FileUtils.mv(
|
|
285
|
+
file.path,
|
|
286
|
+
path
|
|
287
|
+
)
|
|
288
|
+
path
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# Base path for imported and exported files
|
|
292
|
+
# @param [String]
|
|
293
|
+
# @return [String] the base path for files that this parser will "parse"
|
|
294
|
+
def base_path(type = 'import')
|
|
295
|
+
# account for multiple versions of hyku
|
|
296
|
+
is_multitenant = ENV['HYKU_MULTITENANT'] == 'true' || ENV['SETTINGS__MULTITENANCY__ENABLED'] == 'true'
|
|
297
|
+
is_multitenant ? File.join(Bulkrax.send("#{type}_path"), ::Site.instance.account.name) : Bulkrax.send("#{type}_path")
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
# Path where we'll store the import metadata and files
|
|
301
|
+
# this is used for uploaded and cloud files
|
|
302
|
+
# @return [String]
|
|
303
|
+
def path_for_import
|
|
304
|
+
@path_for_import = File.join(base_path, importerexporter.path_string)
|
|
305
|
+
FileUtils.mkdir_p(@path_for_import) unless File.exist?(@path_for_import)
|
|
306
|
+
@path_for_import
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
# @abstract Subclass and override {#setup_export_file} to implement behavior for the parser.
|
|
310
|
+
def setup_export_file
|
|
311
|
+
raise NotImplementedError, 'must be defined' if exporter?
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
# @abstract Subclass and override {#write_files} to implement behavior for the parser.
|
|
315
|
+
def write_files
|
|
316
|
+
raise NotImplementedError, 'must be defined' if exporter?
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
# @return [TrueClass,FalseClass]
|
|
320
|
+
def importer?
|
|
321
|
+
importerexporter.is_a?(Bulkrax::Importer)
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
# @return [TrueClass,FalseClass]
|
|
325
|
+
def exporter?
|
|
326
|
+
importerexporter.is_a?(Bulkrax::Exporter)
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
# @param limit [Integer] limit set on the importerexporter
|
|
330
|
+
# @param index [Integer] index of current iteration
|
|
331
|
+
# @return [TrueClass,FalseClass]
|
|
332
|
+
def limit_reached?(limit, index)
|
|
333
|
+
return false if limit.nil? || limit.zero? # no limit
|
|
334
|
+
index >= limit
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
# Override to add specific validations
|
|
338
|
+
# @return [TrueClass,FalseClass]
|
|
339
|
+
def valid_import?
|
|
340
|
+
true
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
# @return [TrueClass,FalseClass]
|
|
344
|
+
def record_has_source_identifier(record, index)
|
|
345
|
+
if record[source_identifier].blank?
|
|
346
|
+
if Bulkrax.fill_in_blank_source_identifiers.present?
|
|
347
|
+
record[source_identifier] = Bulkrax.fill_in_blank_source_identifiers.call(self, index)
|
|
348
|
+
else
|
|
349
|
+
invalid_record("Missing #{source_identifier} for #{record.to_h}\n")
|
|
350
|
+
false
|
|
351
|
+
end
|
|
352
|
+
else
|
|
353
|
+
true
|
|
354
|
+
end
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
# rubocop:disable Rails/SkipsModelValidations
|
|
358
|
+
def invalid_record(message)
|
|
359
|
+
current_run.invalid_records ||= ""
|
|
360
|
+
current_run.invalid_records += message
|
|
361
|
+
current_run.save
|
|
362
|
+
ImporterRun.increment_counter(:failed_records, current_run.id)
|
|
363
|
+
ImporterRun.decrement_counter(:enqueued_records, current_run.id) unless ImporterRun.find(current_run.id).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
|
|
364
|
+
end
|
|
365
|
+
# rubocop:enable Rails/SkipsModelValidations
|
|
366
|
+
|
|
367
|
+
# @return [Array<String>]
|
|
368
|
+
def required_elements
|
|
369
|
+
matched_elements = ((importerexporter.mapping.keys || []) & (Bulkrax.required_elements || []))
|
|
370
|
+
unless matched_elements.count == Bulkrax.required_elements.count
|
|
371
|
+
missing_elements = Bulkrax.required_elements - matched_elements
|
|
372
|
+
error_alert = "Missing mapping for at least one required element, missing mappings are: #{missing_elements.join(', ')}"
|
|
373
|
+
raise StandardError, error_alert
|
|
374
|
+
end
|
|
375
|
+
if Bulkrax.fill_in_blank_source_identifiers
|
|
376
|
+
Bulkrax.required_elements
|
|
377
|
+
else
|
|
378
|
+
Bulkrax.required_elements + [source_identifier]
|
|
379
|
+
end
|
|
380
|
+
end
|
|
381
|
+
|
|
382
|
+
def new_entry(entryclass, type)
|
|
383
|
+
entryclass.new(
|
|
384
|
+
importerexporter_id: importerexporter.id,
|
|
385
|
+
importerexporter_type: type
|
|
386
|
+
)
|
|
387
|
+
end
|
|
388
|
+
|
|
389
|
+
def find_or_create_entry(entryclass, identifier, type, raw_metadata = nil)
|
|
390
|
+
# limit entry search to just this importer or exporter. Don't go moving them
|
|
391
|
+
entry = importerexporter.entries.where(
|
|
392
|
+
identifier: identifier
|
|
393
|
+
).first
|
|
394
|
+
entry ||= entryclass.new(
|
|
395
|
+
importerexporter_id: importerexporter.id,
|
|
396
|
+
importerexporter_type: type,
|
|
397
|
+
identifier: identifier
|
|
398
|
+
)
|
|
399
|
+
entry.raw_metadata = raw_metadata
|
|
400
|
+
# Setting parsed_metadata specifically for the id so we can find the object via the
|
|
401
|
+
# id in a delete. This is likely to get clobbered in a regular import, which is fine.
|
|
402
|
+
entry.parsed_metadata = { id: raw_metadata['id'] } if raw_metadata&.key?('id')
|
|
403
|
+
entry.save!
|
|
404
|
+
entry
|
|
405
|
+
end
|
|
406
|
+
|
|
407
|
+
# @todo - review this method - is it ever used?
|
|
408
|
+
def record(identifier, _opts = {})
|
|
409
|
+
return @record if @record
|
|
410
|
+
|
|
411
|
+
@record = entry_class.new(self, identifier)
|
|
412
|
+
@record.build
|
|
413
|
+
return @record
|
|
414
|
+
end
|
|
415
|
+
|
|
416
|
+
def total
|
|
417
|
+
0
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
def collections_total
|
|
421
|
+
0
|
|
422
|
+
end
|
|
423
|
+
|
|
424
|
+
def file_sets_total
|
|
425
|
+
0
|
|
426
|
+
end
|
|
427
|
+
|
|
428
|
+
def write
|
|
429
|
+
write_files
|
|
430
|
+
zip
|
|
431
|
+
end
|
|
432
|
+
|
|
433
|
+
# Extracts a zip verbatim into {#importer_unzip_path}, preserving the zip's
|
|
434
|
+
# internal structure. Filters macOS junk (`__MACOSX/`, `.DS_Store`, `._*`).
|
|
435
|
+
# Parser subclasses that need to interpret the zip's structure (e.g.
|
|
436
|
+
# {Bulkrax::CsvParser#unzip_with_primary_csv}) should call a more specific
|
|
437
|
+
# method rather than this one.
|
|
438
|
+
def unzip(file_to_unzip)
|
|
439
|
+
return untar(file_to_unzip) if file_to_unzip.end_with?('.tar.gz')
|
|
440
|
+
|
|
441
|
+
dest_dir = importer_unzip_path(mkdir: true)
|
|
442
|
+
Zip::File.open(file_to_unzip) do |zip_file|
|
|
443
|
+
zip_file.each do |entry|
|
|
444
|
+
next unless entry.file?
|
|
445
|
+
next if macos_junk_entry?(entry.name)
|
|
446
|
+
reject_unsafe_entry!(entry.name)
|
|
447
|
+
dest_path = safe_extract_path(dest_dir, entry.name)
|
|
448
|
+
FileUtils.mkdir_p(File.dirname(dest_path))
|
|
449
|
+
next if File.exist?(dest_path)
|
|
450
|
+
extract_zip_entry(zip_file, entry, dest_dir, entry.name, dest_path)
|
|
451
|
+
end
|
|
452
|
+
end
|
|
453
|
+
end
|
|
454
|
+
|
|
455
|
+
# rubyzip 2.x: extract(entry, absolute_dest_path)
|
|
456
|
+
# rubyzip 3.x: extract(entry, relative_name, destination_directory: dir)
|
|
457
|
+
#
|
|
458
|
+
# Callers are responsible for passing a `dest_path` produced by
|
|
459
|
+
# {#safe_extract_path} so the write can't escape `dest_dir`.
|
|
460
|
+
def extract_zip_entry(zip_file, entry, dest_dir, relative_name, dest_path)
|
|
461
|
+
if zip_file.method(:extract).arity == 2
|
|
462
|
+
zip_file.extract(entry, dest_path)
|
|
463
|
+
else
|
|
464
|
+
zip_file.extract(entry, relative_name, destination_directory: dest_dir)
|
|
465
|
+
end
|
|
466
|
+
end
|
|
467
|
+
|
|
468
|
+
def macos_junk_entry?(name)
|
|
469
|
+
name.start_with?('__MACOSX/') || name.split('/').any? { |part| part == '.DS_Store' || part.start_with?('._') }
|
|
470
|
+
end
|
|
471
|
+
|
|
472
|
+
# Zip Slip preflight — reject entries whose names are obviously unsafe
|
|
473
|
+
# (absolute paths, `..` segments) before we touch the filesystem.
|
|
474
|
+
# {#safe_extract_path} is the final line of defense; this check just
|
|
475
|
+
# fails fast with a clear message. Delegates to {Bulkrax::SafePath} so
|
|
476
|
+
# cloud-file destinations and sources share the same rules.
|
|
477
|
+
#
|
|
478
|
+
# @raise [Bulkrax::UnzipError] if the entry name is unsafe
|
|
479
|
+
<<<<<<< HEAD
|
|
480
|
+
def reject_unsafe_entry!(name)
|
|
481
|
+
return unless name.start_with?('/') || name.split('/').include?('..')
|
|
482
|
+
raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: name)
|
|
483
|
+
=======
|
|
484
|
+
def self.reject_unsafe_entry!(name)
|
|
485
|
+
Bulkrax::SafePath.reject_unsafe_entry!(name)
|
|
486
|
+
rescue Bulkrax::UnsafePathError
|
|
487
|
+
raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: name.dup.force_encoding(Encoding::UTF_8).scrub)
|
|
488
|
+
>>>>>>> acc18e45 (Merge commit from fork)
|
|
489
|
+
end
|
|
490
|
+
|
|
491
|
+
# Zip Slip chokepoint. Resolves `relative_dest` against `dest_dir` and
|
|
492
|
+
# returns the absolute destination path — but only if it stays inside
|
|
493
|
+
# `dest_dir`, including through any symlink. Callers must use this
|
|
494
|
+
# value rather than building their own path with `File.join`, so the
|
|
495
|
+
# path returned is always safe by construction.
|
|
496
|
+
#
|
|
497
|
+
# @return [String] absolute destination path, validated to be inside `dest_dir`
|
|
498
|
+
# @raise [Bulkrax::UnzipError] if `relative_dest` escapes `dest_dir`
|
|
499
|
+
def safe_extract_path(dest_dir, relative_dest)
|
|
500
|
+
Bulkrax::SafePath.confine!(dest_dir, relative_dest)
|
|
501
|
+
rescue Bulkrax::UnsafePathError
|
|
502
|
+
raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: relative_dest)
|
|
503
|
+
end
|
|
504
|
+
|
|
505
|
+
def copy_file(file_to_copy)
|
|
506
|
+
destination = File.join(importer_unzip_path(mkdir: true), File.basename(file_to_copy))
|
|
507
|
+
FileUtils.cp(file_to_copy, destination)
|
|
508
|
+
end
|
|
509
|
+
|
|
510
|
+
def untar(file_to_untar)
|
|
511
|
+
Dir.mkdir(importer_unzip_path(mkdir: true)) unless File.directory?(importer_unzip_path(mkdir: true))
|
|
512
|
+
command = "tar -xzf #{Shellwords.escape(file_to_untar)} -C #{Shellwords.escape(importer_unzip_path)}"
|
|
513
|
+
result = system(command)
|
|
514
|
+
raise "Failed to extract #{file_to_untar}" unless result
|
|
515
|
+
end
|
|
516
|
+
|
|
517
|
+
def zip
|
|
518
|
+
FileUtils.mkdir_p(exporter_export_zip_path)
|
|
519
|
+
|
|
520
|
+
Dir["#{exporter_export_path}/**"].each do |folder|
|
|
521
|
+
zip_path = "#{exporter_export_zip_path.split('/').last}_#{folder.split('/').last}.zip"
|
|
522
|
+
FileUtils.rm_rf("#{exporter_export_zip_path}/#{zip_path}")
|
|
523
|
+
|
|
524
|
+
Zip::File.open(File.join("#{exporter_export_zip_path}/#{zip_path}"), create: true) do |zip_file|
|
|
525
|
+
Dir["#{folder}/**/**"].each do |file|
|
|
526
|
+
zip_file.add(file.sub("#{folder}/", ''), file)
|
|
527
|
+
end
|
|
528
|
+
end
|
|
529
|
+
end
|
|
530
|
+
end
|
|
531
|
+
|
|
532
|
+
# Path for the import
|
|
533
|
+
# @return [String]
|
|
534
|
+
def import_file_path
|
|
535
|
+
@import_file_path ||= real_import_file_path
|
|
536
|
+
end
|
|
537
|
+
|
|
538
|
+
private
|
|
539
|
+
|
|
540
|
+
# @return [String]
|
|
541
|
+
def real_import_file_path
|
|
542
|
+
parser_fields['import_file_path']
|
|
543
|
+
end
|
|
544
|
+
end
|
|
545
|
+
end
|
|
@@ -178,12 +178,19 @@ unless ENV.fetch('BULKRAX_NO_BAGIT', 'false').to_s == 'true'
|
|
|
178
178
|
# @todo - investigate getting directory structure
|
|
179
179
|
# @todo - investigate using perform_later, and having the importer check for
|
|
180
180
|
# DownloadCloudFileJob before it starts
|
|
181
|
-
def retrieve_cloud_files(files,
|
|
181
|
+
def retrieve_cloud_files(files, importer)
|
|
182
182
|
# There should only be one zip file for Bagit, take the first
|
|
183
183
|
return if files['0'].blank?
|
|
184
|
-
|
|
184
|
+
|
|
185
|
+
file = files['0']
|
|
186
|
+
# Checked on the raw file_name before any normalization — see
|
|
187
|
+
# Bulkrax::CsvParser#plan_cloud_file for why this must come first.
|
|
188
|
+
Bulkrax::CloudSourceValidator.preflight!(file)
|
|
189
|
+
|
|
190
|
+
relative_filename = file['file_name'].tr(' ', '_')
|
|
191
|
+
target_file = Bulkrax::SafePath.confine!(path_for_import, relative_filename, require_file: true)
|
|
185
192
|
# Now because we want the files in place before the importer runs
|
|
186
|
-
Bulkrax::DownloadCloudFileJob.perform_now(
|
|
193
|
+
Bulkrax::DownloadCloudFileJob.perform_now(file, importer_id: importer.id, relative_filename: relative_filename)
|
|
187
194
|
return target_file
|
|
188
195
|
end
|
|
189
196
|
|
|
@@ -209,24 +209,55 @@ module Bulkrax
|
|
|
209
209
|
# @todo - investigate using perform_later, and having the importer check for
|
|
210
210
|
# DownloadCloudFileJob before it starts
|
|
211
211
|
def retrieve_cloud_files(files, importer)
|
|
212
|
+
plan = plan_cloud_files(files)
|
|
213
|
+
dispatch_cloud_files!(plan, importer)
|
|
214
|
+
importer[:parser_fields]['original_file_paths'] = plan.map { |_file, _relative_filename, target_file| target_file }
|
|
215
|
+
return nil
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# Validates and resolves the whole batch, without enqueueing anything
|
|
219
|
+
# yet. Otherwise an earlier, valid file's job would already be running
|
|
220
|
+
# by the time a later file in the same batch is found to be unsafe.
|
|
221
|
+
# Split out from {#retrieve_cloud_files} so a caller (the importers
|
|
222
|
+
# controller) can persist the resolved paths before dispatching.
|
|
223
|
+
#
|
|
224
|
+
# @return [Array<Array(Hash, String, String)>] one entry per file: the
|
|
225
|
+
# file spec, its relative destination filename, and the resolved
|
|
226
|
+
# absolute target path
|
|
227
|
+
def plan_cloud_files(files)
|
|
212
228
|
files_path = File.join(path_for_import, 'files')
|
|
213
229
|
FileUtils.mkdir_p(files_path) unless File.exist?(files_path)
|
|
214
|
-
|
|
215
|
-
files.
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
230
|
+
|
|
231
|
+
files.values.map { |file| plan_cloud_file(file) }
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# Enqueues a download job for each entry of an already-validated plan
|
|
235
|
+
# (see {#plan_cloud_files}).
|
|
236
|
+
def dispatch_cloud_files!(plan, importer)
|
|
237
|
+
plan.each { |file, relative_filename, _target_file| Bulkrax::DownloadCloudFileJob.perform_later(file, importer_id: importer.id, relative_filename: relative_filename) }
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# Validates one cloud-file spec and resolves its destination, without
|
|
241
|
+
# enqueueing anything yet.
|
|
242
|
+
#
|
|
243
|
+
# @return [Array(Hash, String, String)] the file spec, its relative
|
|
244
|
+
# destination filename, and the resolved absolute target path
|
|
245
|
+
def plan_cloud_file(file)
|
|
246
|
+
# fixes bug where auth headers do not get attached properly
|
|
247
|
+
if file['auth_header'].present?
|
|
248
|
+
file['headers'] ||= {}
|
|
249
|
+
file['headers'].merge!(file['auth_header'])
|
|
227
250
|
end
|
|
228
|
-
|
|
229
|
-
|
|
251
|
+
|
|
252
|
+
# Checked on the raw file_name, before it's joined under 'files/':
|
|
253
|
+
# that join would otherwise neutralize a leading `/` (File.join
|
|
254
|
+
# doesn't treat it as absolute once it's not the first argument).
|
|
255
|
+
Bulkrax::CloudSourceValidator.preflight!(file)
|
|
256
|
+
|
|
257
|
+
# this only works for uniquely named files
|
|
258
|
+
relative_filename = File.join('files', file['file_name'].tr(' ', '_'))
|
|
259
|
+
target_file = Bulkrax::SafePath.confine!(path_for_import, relative_filename, require_file: true)
|
|
260
|
+
[file, relative_filename, target_file]
|
|
230
261
|
end
|
|
231
262
|
|
|
232
263
|
# export methods
|