bulkrax 8.2.4 → 8.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/javascripts/bulkrax/bulkrax.js +9 -7
- data/app/assets/javascripts/bulkrax/importers.js.erb +134 -99
- data/app/controllers/bulkrax/importers_controller.rb +28 -112
- data/app/controllers/concerns/bulkrax/datatables_behavior.rb +1 -1
- data/app/factories/bulkrax/object_factory.rb +1 -4
- data/app/jobs/bulkrax/download_cloud_file_job.rb +21 -156
- data/app/jobs/bulkrax/export_work_job.rb +1 -1
- data/app/models/bulkrax/status.rb +1 -1
- data/app/models/concerns/bulkrax/export_behavior.rb +4 -8
- data/app/parsers/bulkrax/application_parser.rb +2 -25
- data/app/parsers/bulkrax/bagit_parser.rb +3 -10
- data/app/parsers/bulkrax/csv_parser.rb +17 -51
- data/app/parsers/bulkrax/parser_export_record_set.rb +1 -1
- data/app/views/bulkrax/importers/_bagit_fields.html.erb +10 -3
- data/app/views/bulkrax/importers/_csv_fields.html.erb +13 -9
- data/app/views/bulkrax/importers/_file_uploader.html.erb +37 -0
- data/app/views/bulkrax/importers/_xml_fields.html.erb +10 -4
- data/config/locales/bulkrax.en.yml +0 -3
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +0 -1
- data/lib/tasks/bulkrax_tasks.rake +1 -1
- metadata +9 -32
- data/app/errors/bulkrax/unsafe_cloud_source_error.rb +0 -11
- data/app/errors/bulkrax/unsafe_path_error.rb +0 -8
- data/app/parsers/bulkrax/application_parser.rb.orig +0 -545
- data/app/services/bulkrax/cloud_source_validator.rb +0 -123
- data/app/services/bulkrax/safe_path.rb +0 -142
- data/db/migrate/20241203010707_entry_error_denormalization.rb +0 -7
- data/db/migrate/20241205212513_faster_first_entry.rb +0 -7
metadata
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: bulkrax
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 8.
|
|
4
|
+
version: 8.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Rob Kaufman
|
|
8
|
+
autorequire:
|
|
8
9
|
bindir: bin
|
|
9
10
|
cert_chain: []
|
|
10
|
-
date:
|
|
11
|
+
date: 2024-12-02 00:00:00.000000000 Z
|
|
11
12
|
dependencies:
|
|
12
13
|
- !ruby/object:Gem::Dependency
|
|
13
14
|
name: rails
|
|
@@ -29,14 +30,14 @@ dependencies:
|
|
|
29
30
|
requirements:
|
|
30
31
|
- - "~>"
|
|
31
32
|
- !ruby/object:Gem::Version
|
|
32
|
-
version: 0.6
|
|
33
|
+
version: 0.4.6
|
|
33
34
|
type: :runtime
|
|
34
35
|
prerelease: false
|
|
35
36
|
version_requirements: !ruby/object:Gem::Requirement
|
|
36
37
|
requirements:
|
|
37
38
|
- - "~>"
|
|
38
39
|
- !ruby/object:Gem::Version
|
|
39
|
-
version: 0.6
|
|
40
|
+
version: 0.4.6
|
|
40
41
|
- !ruby/object:Gem::Dependency
|
|
41
42
|
name: coderay
|
|
42
43
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -237,26 +238,6 @@ dependencies:
|
|
|
237
238
|
- - ">="
|
|
238
239
|
- !ruby/object:Gem::Version
|
|
239
240
|
version: '0'
|
|
240
|
-
- !ruby/object:Gem::Dependency
|
|
241
|
-
name: ssrf_filter
|
|
242
|
-
requirement: !ruby/object:Gem::Requirement
|
|
243
|
-
requirements:
|
|
244
|
-
- - ">="
|
|
245
|
-
- !ruby/object:Gem::Version
|
|
246
|
-
version: '1.0'
|
|
247
|
-
- - "<"
|
|
248
|
-
- !ruby/object:Gem::Version
|
|
249
|
-
version: '2.0'
|
|
250
|
-
type: :runtime
|
|
251
|
-
prerelease: false
|
|
252
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
253
|
-
requirements:
|
|
254
|
-
- - ">="
|
|
255
|
-
- !ruby/object:Gem::Version
|
|
256
|
-
version: '1.0'
|
|
257
|
-
- - "<"
|
|
258
|
-
- !ruby/object:Gem::Version
|
|
259
|
-
version: '2.0'
|
|
260
241
|
- !ruby/object:Gem::Dependency
|
|
261
242
|
name: dry-monads
|
|
262
243
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -361,8 +342,6 @@ files:
|
|
|
361
342
|
- app/controllers/concerns/bulkrax/api.rb
|
|
362
343
|
- app/controllers/concerns/bulkrax/datatables_behavior.rb
|
|
363
344
|
- app/controllers/concerns/bulkrax/download_behavior.rb
|
|
364
|
-
- app/errors/bulkrax/unsafe_cloud_source_error.rb
|
|
365
|
-
- app/errors/bulkrax/unsafe_path_error.rb
|
|
366
345
|
- app/factories/bulkrax/object_factory.rb
|
|
367
346
|
- app/factories/bulkrax/object_factory_interface.rb
|
|
368
347
|
- app/factories/bulkrax/valkyrie_object_factory.rb
|
|
@@ -425,17 +404,14 @@ files:
|
|
|
425
404
|
- app/models/concerns/bulkrax/importer_exporter_behavior.rb
|
|
426
405
|
- app/models/concerns/bulkrax/status_info.rb
|
|
427
406
|
- app/parsers/bulkrax/application_parser.rb
|
|
428
|
-
- app/parsers/bulkrax/application_parser.rb.orig
|
|
429
407
|
- app/parsers/bulkrax/bagit_parser.rb
|
|
430
408
|
- app/parsers/bulkrax/csv_parser.rb
|
|
431
409
|
- app/parsers/bulkrax/oai_dc_parser.rb
|
|
432
410
|
- app/parsers/bulkrax/oai_qualified_dc_parser.rb
|
|
433
411
|
- app/parsers/bulkrax/parser_export_record_set.rb
|
|
434
412
|
- app/parsers/bulkrax/xml_parser.rb
|
|
435
|
-
- app/services/bulkrax/cloud_source_validator.rb
|
|
436
413
|
- app/services/bulkrax/factory_class_finder.rb
|
|
437
414
|
- app/services/bulkrax/remove_relationships_for_importer.rb
|
|
438
|
-
- app/services/bulkrax/safe_path.rb
|
|
439
415
|
- app/services/hyrax/custom_queries/find_by_source_identifier.rb
|
|
440
416
|
- app/services/wings/custom_queries/find_by_source_identifier.rb
|
|
441
417
|
- app/views/bulkrax/entries/_parsed_metadata.html.erb
|
|
@@ -452,6 +428,7 @@ files:
|
|
|
452
428
|
- app/views/bulkrax/importers/_csv_fields.html.erb
|
|
453
429
|
- app/views/bulkrax/importers/_edit_form_buttons.html.erb
|
|
454
430
|
- app/views/bulkrax/importers/_edit_item_buttons.html.erb
|
|
431
|
+
- app/views/bulkrax/importers/_file_uploader.html.erb
|
|
455
432
|
- app/views/bulkrax/importers/_form.html.erb
|
|
456
433
|
- app/views/bulkrax/importers/_oai_fields.html.erb
|
|
457
434
|
- app/views/bulkrax/importers/_xml_fields.html.erb
|
|
@@ -510,8 +487,6 @@ files:
|
|
|
510
487
|
- db/migrate/20240823173525_add_error_tracking_to_pending_relationships.rb
|
|
511
488
|
- db/migrate/20240916182737_add_last_imported_at_to_bulkrax_importers.rb
|
|
512
489
|
- db/migrate/20240916182823_add_next_import_at_to_bulkrax_importers.rb
|
|
513
|
-
- db/migrate/20241203010707_entry_error_denormalization.rb
|
|
514
|
-
- db/migrate/20241205212513_faster_first_entry.rb
|
|
515
490
|
- lib/bulkrax.rb
|
|
516
491
|
- lib/bulkrax/engine.rb
|
|
517
492
|
- lib/bulkrax/entry_spec_helper.rb
|
|
@@ -529,6 +504,7 @@ homepage: https://github.com/samvera-labs/bulkrax
|
|
|
529
504
|
licenses:
|
|
530
505
|
- Apache-2.0
|
|
531
506
|
metadata: {}
|
|
507
|
+
post_install_message:
|
|
532
508
|
rdoc_options: []
|
|
533
509
|
require_paths:
|
|
534
510
|
- lib
|
|
@@ -543,7 +519,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
543
519
|
- !ruby/object:Gem::Version
|
|
544
520
|
version: '0'
|
|
545
521
|
requirements: []
|
|
546
|
-
rubygems_version: 3.
|
|
522
|
+
rubygems_version: 3.4.20
|
|
523
|
+
signing_key:
|
|
547
524
|
specification_version: 4
|
|
548
525
|
summary: Import and export tool for Hyrax and Hyku
|
|
549
526
|
test_files: []
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Bulkrax
|
|
4
|
-
# Raised by {Bulkrax::CloudSourceValidator} when a cloud-file source
|
|
5
|
-
# (the `url` a user selected via browse-everything, or supplied directly
|
|
6
|
-
# in importer params) can't be safely downloaded: a scheme other than
|
|
7
|
-
# `file`/`http`/`https`, a `file:` source outside the configured
|
|
8
|
-
# filesystem-provider root, or an HTTP(S) source that resolves to a
|
|
9
|
-
# private, loopback, or link-local/metadata address.
|
|
10
|
-
class UnsafeCloudSourceError < StandardError; end
|
|
11
|
-
end
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Bulkrax
|
|
4
|
-
# Raised by {Bulkrax::SafePath} when a path would escape its configured
|
|
5
|
-
# root, whether that root is a ZIP extraction directory, an importer's
|
|
6
|
-
# trusted workspace, or a configured filesystem-provider home.
|
|
7
|
-
class UnsafePathError < StandardError; end
|
|
8
|
-
end
|
|
@@ -1,545 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Bulkrax
|
|
4
|
-
# An abstract class that establishes the API for Bulkrax's import and export parsing.
|
|
5
|
-
#
|
|
6
|
-
# @abstract Subclass the Bulkrax::ApplicationParser to create a parser that handles a specific format (e.g. CSV, Bagit, XML, etc).
|
|
7
|
-
class ApplicationParser # rubocop:disable Metrics/ClassLength
|
|
8
|
-
attr_accessor :importerexporter, :headers
|
|
9
|
-
alias importer importerexporter
|
|
10
|
-
alias exporter importerexporter
|
|
11
|
-
delegate :only_updates, :limit, :current_run, :errors, :mapping,
|
|
12
|
-
:seen, :increment_counters, :parser_fields, :user, :keys_without_numbers,
|
|
13
|
-
:key_without_numbers, :status, :set_status_info, :status_info, :status_at,
|
|
14
|
-
:exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
|
|
15
|
-
:zip?, :file?, :remove_and_rerun, :zip_file?,
|
|
16
|
-
to: :importerexporter
|
|
17
|
-
|
|
18
|
-
# @todo Convert to `class_attribute :parser_fiels, default: {}`
|
|
19
|
-
def self.parser_fields
|
|
20
|
-
{}
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
# @return [TrueClass,FalseClass] this parser does or does not support exports.
|
|
24
|
-
#
|
|
25
|
-
# @todo Convert to `class_attribute :export_supported, default: false, instance_predicate: true` and `self << class; alias export_supported? export_supported; end`
|
|
26
|
-
def self.export_supported?
|
|
27
|
-
false
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
# @return [TrueClass,FalseClass] this parser does or does not support imports.
|
|
31
|
-
#
|
|
32
|
-
# @todo Convert to `class_attribute :import_supported, default: false, instance_predicate: true` and `self << class; alias import_supported? import_supported; end`
|
|
33
|
-
def self.import_supported?
|
|
34
|
-
true
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
def initialize(importerexporter)
|
|
38
|
-
@importerexporter = importerexporter
|
|
39
|
-
@headers = []
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
# @api public
|
|
43
|
-
# @abstract Subclass and override {#entry_class} to implement behavior for the parser.
|
|
44
|
-
def entry_class
|
|
45
|
-
raise NotImplementedError, 'must be defined'
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
def work_entry_class
|
|
49
|
-
entry_class
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
# @api public
|
|
53
|
-
# @abstract Subclass and override {#collection_entry_class} to implement behavior for the parser.
|
|
54
|
-
def collection_entry_class
|
|
55
|
-
raise NotImplementedError, 'must be defined'
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
# @api public
|
|
59
|
-
# @abstract Subclass and override {#file_set_entry_class} to implement behavior for the parser.
|
|
60
|
-
def file_set_entry_class
|
|
61
|
-
raise NotImplementedError, 'must be defined'
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
# @api public
|
|
65
|
-
# @abstract Subclass and override {#records} to implement behavior for the parser.
|
|
66
|
-
def records(_opts = {})
|
|
67
|
-
raise NotImplementedError, 'must be defined'
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
# @return [Symbol] the name of the identifying property in the source system from which we're
|
|
71
|
-
# importing (e.g. is *not* this application that mounts *this* Bulkrax engine).
|
|
72
|
-
#
|
|
73
|
-
# @see #work_identifier
|
|
74
|
-
# @see https://github.com/samvera-labs/bulkrax/wiki/CSV-Importer#source-identifier Bulkrax Wiki regarding source identifier
|
|
75
|
-
def source_identifier
|
|
76
|
-
@source_identifier ||= get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('from')&.first&.to_sym || :source_identifier
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
# @return [Symbol] the name of the identifying property for the system which we're importing
|
|
80
|
-
# into (e.g. the application that mounts *this* Bulkrax engine)
|
|
81
|
-
# @see #source_identifier
|
|
82
|
-
def work_identifier
|
|
83
|
-
@work_identifier ||= get_field_mapping_hash_for('source_identifier')&.keys&.first&.to_sym || :source
|
|
84
|
-
end
|
|
85
|
-
|
|
86
|
-
# @return [Symbol] the solr property of the source_identifier. Used for searching.
|
|
87
|
-
# defaults to work_identifier value + "_sim"
|
|
88
|
-
# @see #work_identifier
|
|
89
|
-
def work_identifier_search_field
|
|
90
|
-
@work_identifier_search_field ||= Array.wrap(get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('search_field'))&.first&.to_s || "#{work_identifier}_sim"
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
# @return [String]
|
|
94
|
-
def generated_metadata_mapping
|
|
95
|
-
@generated_metadata_mapping ||= 'generated'
|
|
96
|
-
end
|
|
97
|
-
|
|
98
|
-
# @return [String, NilClass]
|
|
99
|
-
# @see #related_parents_raw_mapping
|
|
100
|
-
def related_parents_raw_mapping
|
|
101
|
-
@related_parents_raw_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.values&.first&.[]('from')&.first
|
|
102
|
-
end
|
|
103
|
-
|
|
104
|
-
# @return [String]
|
|
105
|
-
# @see #related_parents_field_mapping
|
|
106
|
-
def related_parents_parsed_mapping
|
|
107
|
-
@related_parents_parsed_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.keys&.first || 'parents'
|
|
108
|
-
end
|
|
109
|
-
|
|
110
|
-
# @return [String, NilClass]
|
|
111
|
-
# @see #related_children_parsed_mapping
|
|
112
|
-
def related_children_raw_mapping
|
|
113
|
-
@related_children_raw_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.values&.first&.[]('from')&.first
|
|
114
|
-
end
|
|
115
|
-
|
|
116
|
-
# @return [String]
|
|
117
|
-
# @see #related_children_raw_mapping
|
|
118
|
-
def related_children_parsed_mapping
|
|
119
|
-
@related_children_parsed_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.keys&.first || 'children'
|
|
120
|
-
end
|
|
121
|
-
|
|
122
|
-
# @api private
|
|
123
|
-
def get_field_mapping_hash_for(key)
|
|
124
|
-
return instance_variable_get("@#{key}_hash") if instance_variable_get("@#{key}_hash").present?
|
|
125
|
-
|
|
126
|
-
mapping = importerexporter.field_mapping.is_a?(Hash) ? importerexporter.field_mapping : {}
|
|
127
|
-
instance_variable_set(
|
|
128
|
-
"@#{key}_hash",
|
|
129
|
-
mapping&.with_indifferent_access&.select { |_, h| h.key?(key) }
|
|
130
|
-
)
|
|
131
|
-
raise StandardError, "more than one #{key} declared: #{instance_variable_get("@#{key}_hash").keys.join(', ')}" if instance_variable_get("@#{key}_hash").length > 1
|
|
132
|
-
|
|
133
|
-
instance_variable_get("@#{key}_hash")
|
|
134
|
-
end
|
|
135
|
-
|
|
136
|
-
# @return [Array<String>]
|
|
137
|
-
def model_field_mappings
|
|
138
|
-
model_mappings = Bulkrax.field_mappings[self.class.to_s]&.dig('model', :from) || []
|
|
139
|
-
model_mappings |= ['model']
|
|
140
|
-
|
|
141
|
-
model_mappings
|
|
142
|
-
end
|
|
143
|
-
|
|
144
|
-
# @return [String]
|
|
145
|
-
def perform_method
|
|
146
|
-
if self.validate_only
|
|
147
|
-
'perform_now'
|
|
148
|
-
else
|
|
149
|
-
'perform_later'
|
|
150
|
-
end
|
|
151
|
-
end
|
|
152
|
-
|
|
153
|
-
# The visibility of the record. Acceptable values are: "open", "embargo", "lease", "authenticated", "restricted". The default is "open"
|
|
154
|
-
#
|
|
155
|
-
# @return [String]
|
|
156
|
-
# @see https://github.com/samvera/hydra-head/blob/main/hydra-access-controls/app/models/concerns/hydra/access_controls/access_right.rb Hydra::AccessControls::AccessRight for details on the range of values.
|
|
157
|
-
# @see https://github.com/samvera/hyrax/blob/bd2bcffc33e183904be2c175367648815f25bc2b/app/services/hyrax/visibility_intention.rb Hyrax::VisibilityIntention for how we process the visibility.
|
|
158
|
-
def visibility
|
|
159
|
-
@visibility ||= self.parser_fields['visibility'] || 'open'
|
|
160
|
-
end
|
|
161
|
-
|
|
162
|
-
def create_collections
|
|
163
|
-
create_objects(['collection'])
|
|
164
|
-
end
|
|
165
|
-
|
|
166
|
-
def create_works
|
|
167
|
-
create_objects(['work'])
|
|
168
|
-
end
|
|
169
|
-
|
|
170
|
-
def create_file_sets
|
|
171
|
-
create_objects(['file_set'])
|
|
172
|
-
end
|
|
173
|
-
|
|
174
|
-
def create_relationships
|
|
175
|
-
create_objects(['relationship'])
|
|
176
|
-
end
|
|
177
|
-
|
|
178
|
-
# @api public
|
|
179
|
-
#
|
|
180
|
-
# @param types [Array<Symbol>] the types of objects that we'll create.
|
|
181
|
-
#
|
|
182
|
-
# @see Bulkrax::Importer::DEFAULT_OBJECT_TYPES
|
|
183
|
-
# @see #create_collections
|
|
184
|
-
# @see #create_works
|
|
185
|
-
# @see #create_file_sets
|
|
186
|
-
# @see #create_relationships
|
|
187
|
-
def create_objects(types_array = nil)
|
|
188
|
-
index = 0
|
|
189
|
-
(types_array || %w[collection work file_set relationship]).each do |type|
|
|
190
|
-
if type.eql?('relationship')
|
|
191
|
-
ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: importerexporter.id)
|
|
192
|
-
next
|
|
193
|
-
end
|
|
194
|
-
send(type.pluralize).each do |current_record|
|
|
195
|
-
next unless record_has_source_identifier(current_record, index)
|
|
196
|
-
break if limit_reached?(limit, index)
|
|
197
|
-
seen[current_record[source_identifier]] = true
|
|
198
|
-
create_entry_and_job(current_record, type)
|
|
199
|
-
increment_counters(index, "#{type}": true)
|
|
200
|
-
index += 1
|
|
201
|
-
end
|
|
202
|
-
importer.record_status
|
|
203
|
-
end
|
|
204
|
-
true
|
|
205
|
-
rescue StandardError => e
|
|
206
|
-
set_status_info(e)
|
|
207
|
-
end
|
|
208
|
-
|
|
209
|
-
def rebuild_entries(types_array = nil)
|
|
210
|
-
index = 0
|
|
211
|
-
(types_array || %w[collection work file_set relationship]).each do |type|
|
|
212
|
-
# works are not guaranteed to have Work in the type
|
|
213
|
-
if type.eql?('relationship')
|
|
214
|
-
ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: importerexporter.id)
|
|
215
|
-
next
|
|
216
|
-
end
|
|
217
|
-
importer.entries.where(rebuild_entry_query(type, parser_fields['entry_statuses'])).find_each do |e|
|
|
218
|
-
seen[e.identifier] = true
|
|
219
|
-
e.status_info('Pending', importer.current_run)
|
|
220
|
-
if remove_and_rerun
|
|
221
|
-
delay = calculate_type_delay(type)
|
|
222
|
-
"Bulkrax::DeleteAndImport#{type.camelize}Job".constantize.set(wait: delay).send(perform_method, e, current_run)
|
|
223
|
-
else
|
|
224
|
-
"Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, e.id, current_run.id)
|
|
225
|
-
end
|
|
226
|
-
increment_counters(index)
|
|
227
|
-
index += 1
|
|
228
|
-
end
|
|
229
|
-
end
|
|
230
|
-
end
|
|
231
|
-
|
|
232
|
-
def rebuild_entry_query(type, statuses)
|
|
233
|
-
type_col = Bulkrax::Entry.arel_table['type']
|
|
234
|
-
status_col = Bulkrax::Entry.arel_table['status_message']
|
|
235
|
-
|
|
236
|
-
query = (type == 'work' ? type_col.does_not_match_all(%w[collection file_set]) : type_col.matches(type.camelize))
|
|
237
|
-
query.and(status_col.in(statuses))
|
|
238
|
-
end
|
|
239
|
-
|
|
240
|
-
def calculate_type_delay(type)
|
|
241
|
-
return 2.minutes if type == 'file_set'
|
|
242
|
-
return 1.minute if type == 'work'
|
|
243
|
-
return 0
|
|
244
|
-
end
|
|
245
|
-
|
|
246
|
-
def record_raw_metadata(record)
|
|
247
|
-
record.to_h
|
|
248
|
-
end
|
|
249
|
-
|
|
250
|
-
def record_deleted?(record)
|
|
251
|
-
return false unless record.key?(:delete)
|
|
252
|
-
ActiveModel::Type::Boolean.new.cast(record[:delete])
|
|
253
|
-
end
|
|
254
|
-
|
|
255
|
-
def record_remove_and_rerun?(record)
|
|
256
|
-
return false unless record.key?(:remove_and_rerun)
|
|
257
|
-
ActiveModel::Type::Boolean.new.cast(record[:remove_and_rerun])
|
|
258
|
-
end
|
|
259
|
-
|
|
260
|
-
def create_entry_and_job(current_record, type, identifier = nil)
|
|
261
|
-
identifier ||= current_record[source_identifier]
|
|
262
|
-
new_entry = find_or_create_entry(send("#{type}_entry_class"),
|
|
263
|
-
identifier,
|
|
264
|
-
'Bulkrax::Importer',
|
|
265
|
-
record_raw_metadata(current_record))
|
|
266
|
-
new_entry.status_info('Pending', importer.current_run)
|
|
267
|
-
if record_deleted?(current_record)
|
|
268
|
-
"Bulkrax::Delete#{type.camelize}Job".constantize.send(perform_method, new_entry, current_run)
|
|
269
|
-
elsif record_remove_and_rerun?(current_record) || remove_and_rerun
|
|
270
|
-
delay = calculate_type_delay(type)
|
|
271
|
-
"Bulkrax::DeleteAndImport#{type.camelize}Job".constantize.set(wait: delay).send(perform_method, new_entry, current_run)
|
|
272
|
-
else
|
|
273
|
-
"Bulkrax::Import#{type.camelize}Job".constantize.send(perform_method, new_entry.id, current_run.id)
|
|
274
|
-
end
|
|
275
|
-
end
|
|
276
|
-
|
|
277
|
-
# Optional, define if using browse everything for file upload
|
|
278
|
-
def retrieve_cloud_files(_files, _importer); end
|
|
279
|
-
|
|
280
|
-
# @param file [#path, #original_filename] the file object that with the relevant data for the
|
|
281
|
-
# import.
|
|
282
|
-
def write_import_file(file)
|
|
283
|
-
path = File.join(path_for_import, file.original_filename)
|
|
284
|
-
FileUtils.mv(
|
|
285
|
-
file.path,
|
|
286
|
-
path
|
|
287
|
-
)
|
|
288
|
-
path
|
|
289
|
-
end
|
|
290
|
-
|
|
291
|
-
# Base path for imported and exported files
|
|
292
|
-
# @param [String]
|
|
293
|
-
# @return [String] the base path for files that this parser will "parse"
|
|
294
|
-
def base_path(type = 'import')
|
|
295
|
-
# account for multiple versions of hyku
|
|
296
|
-
is_multitenant = ENV['HYKU_MULTITENANT'] == 'true' || ENV['SETTINGS__MULTITENANCY__ENABLED'] == 'true'
|
|
297
|
-
is_multitenant ? File.join(Bulkrax.send("#{type}_path"), ::Site.instance.account.name) : Bulkrax.send("#{type}_path")
|
|
298
|
-
end
|
|
299
|
-
|
|
300
|
-
# Path where we'll store the import metadata and files
|
|
301
|
-
# this is used for uploaded and cloud files
|
|
302
|
-
# @return [String]
|
|
303
|
-
def path_for_import
|
|
304
|
-
@path_for_import = File.join(base_path, importerexporter.path_string)
|
|
305
|
-
FileUtils.mkdir_p(@path_for_import) unless File.exist?(@path_for_import)
|
|
306
|
-
@path_for_import
|
|
307
|
-
end
|
|
308
|
-
|
|
309
|
-
# @abstract Subclass and override {#setup_export_file} to implement behavior for the parser.
|
|
310
|
-
def setup_export_file
|
|
311
|
-
raise NotImplementedError, 'must be defined' if exporter?
|
|
312
|
-
end
|
|
313
|
-
|
|
314
|
-
# @abstract Subclass and override {#write_files} to implement behavior for the parser.
|
|
315
|
-
def write_files
|
|
316
|
-
raise NotImplementedError, 'must be defined' if exporter?
|
|
317
|
-
end
|
|
318
|
-
|
|
319
|
-
# @return [TrueClass,FalseClass]
|
|
320
|
-
def importer?
|
|
321
|
-
importerexporter.is_a?(Bulkrax::Importer)
|
|
322
|
-
end
|
|
323
|
-
|
|
324
|
-
# @return [TrueClass,FalseClass]
|
|
325
|
-
def exporter?
|
|
326
|
-
importerexporter.is_a?(Bulkrax::Exporter)
|
|
327
|
-
end
|
|
328
|
-
|
|
329
|
-
# @param limit [Integer] limit set on the importerexporter
|
|
330
|
-
# @param index [Integer] index of current iteration
|
|
331
|
-
# @return [TrueClass,FalseClass]
|
|
332
|
-
def limit_reached?(limit, index)
|
|
333
|
-
return false if limit.nil? || limit.zero? # no limit
|
|
334
|
-
index >= limit
|
|
335
|
-
end
|
|
336
|
-
|
|
337
|
-
# Override to add specific validations
|
|
338
|
-
# @return [TrueClass,FalseClass]
|
|
339
|
-
def valid_import?
|
|
340
|
-
true
|
|
341
|
-
end
|
|
342
|
-
|
|
343
|
-
# @return [TrueClass,FalseClass]
|
|
344
|
-
def record_has_source_identifier(record, index)
|
|
345
|
-
if record[source_identifier].blank?
|
|
346
|
-
if Bulkrax.fill_in_blank_source_identifiers.present?
|
|
347
|
-
record[source_identifier] = Bulkrax.fill_in_blank_source_identifiers.call(self, index)
|
|
348
|
-
else
|
|
349
|
-
invalid_record("Missing #{source_identifier} for #{record.to_h}\n")
|
|
350
|
-
false
|
|
351
|
-
end
|
|
352
|
-
else
|
|
353
|
-
true
|
|
354
|
-
end
|
|
355
|
-
end
|
|
356
|
-
|
|
357
|
-
# rubocop:disable Rails/SkipsModelValidations
|
|
358
|
-
def invalid_record(message)
|
|
359
|
-
current_run.invalid_records ||= ""
|
|
360
|
-
current_run.invalid_records += message
|
|
361
|
-
current_run.save
|
|
362
|
-
ImporterRun.increment_counter(:failed_records, current_run.id)
|
|
363
|
-
ImporterRun.decrement_counter(:enqueued_records, current_run.id) unless ImporterRun.find(current_run.id).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
|
|
364
|
-
end
|
|
365
|
-
# rubocop:enable Rails/SkipsModelValidations
|
|
366
|
-
|
|
367
|
-
# @return [Array<String>]
|
|
368
|
-
def required_elements
|
|
369
|
-
matched_elements = ((importerexporter.mapping.keys || []) & (Bulkrax.required_elements || []))
|
|
370
|
-
unless matched_elements.count == Bulkrax.required_elements.count
|
|
371
|
-
missing_elements = Bulkrax.required_elements - matched_elements
|
|
372
|
-
error_alert = "Missing mapping for at least one required element, missing mappings are: #{missing_elements.join(', ')}"
|
|
373
|
-
raise StandardError, error_alert
|
|
374
|
-
end
|
|
375
|
-
if Bulkrax.fill_in_blank_source_identifiers
|
|
376
|
-
Bulkrax.required_elements
|
|
377
|
-
else
|
|
378
|
-
Bulkrax.required_elements + [source_identifier]
|
|
379
|
-
end
|
|
380
|
-
end
|
|
381
|
-
|
|
382
|
-
def new_entry(entryclass, type)
|
|
383
|
-
entryclass.new(
|
|
384
|
-
importerexporter_id: importerexporter.id,
|
|
385
|
-
importerexporter_type: type
|
|
386
|
-
)
|
|
387
|
-
end
|
|
388
|
-
|
|
389
|
-
def find_or_create_entry(entryclass, identifier, type, raw_metadata = nil)
|
|
390
|
-
# limit entry search to just this importer or exporter. Don't go moving them
|
|
391
|
-
entry = importerexporter.entries.where(
|
|
392
|
-
identifier: identifier
|
|
393
|
-
).first
|
|
394
|
-
entry ||= entryclass.new(
|
|
395
|
-
importerexporter_id: importerexporter.id,
|
|
396
|
-
importerexporter_type: type,
|
|
397
|
-
identifier: identifier
|
|
398
|
-
)
|
|
399
|
-
entry.raw_metadata = raw_metadata
|
|
400
|
-
# Setting parsed_metadata specifically for the id so we can find the object via the
|
|
401
|
-
# id in a delete. This is likely to get clobbered in a regular import, which is fine.
|
|
402
|
-
entry.parsed_metadata = { id: raw_metadata['id'] } if raw_metadata&.key?('id')
|
|
403
|
-
entry.save!
|
|
404
|
-
entry
|
|
405
|
-
end
|
|
406
|
-
|
|
407
|
-
# @todo - review this method - is it ever used?
|
|
408
|
-
def record(identifier, _opts = {})
|
|
409
|
-
return @record if @record
|
|
410
|
-
|
|
411
|
-
@record = entry_class.new(self, identifier)
|
|
412
|
-
@record.build
|
|
413
|
-
return @record
|
|
414
|
-
end
|
|
415
|
-
|
|
416
|
-
def total
|
|
417
|
-
0
|
|
418
|
-
end
|
|
419
|
-
|
|
420
|
-
def collections_total
|
|
421
|
-
0
|
|
422
|
-
end
|
|
423
|
-
|
|
424
|
-
def file_sets_total
|
|
425
|
-
0
|
|
426
|
-
end
|
|
427
|
-
|
|
428
|
-
def write
|
|
429
|
-
write_files
|
|
430
|
-
zip
|
|
431
|
-
end
|
|
432
|
-
|
|
433
|
-
# Extracts a zip verbatim into {#importer_unzip_path}, preserving the zip's
|
|
434
|
-
# internal structure. Filters macOS junk (`__MACOSX/`, `.DS_Store`, `._*`).
|
|
435
|
-
# Parser subclasses that need to interpret the zip's structure (e.g.
|
|
436
|
-
# {Bulkrax::CsvParser#unzip_with_primary_csv}) should call a more specific
|
|
437
|
-
# method rather than this one.
|
|
438
|
-
def unzip(file_to_unzip)
|
|
439
|
-
return untar(file_to_unzip) if file_to_unzip.end_with?('.tar.gz')
|
|
440
|
-
|
|
441
|
-
dest_dir = importer_unzip_path(mkdir: true)
|
|
442
|
-
Zip::File.open(file_to_unzip) do |zip_file|
|
|
443
|
-
zip_file.each do |entry|
|
|
444
|
-
next unless entry.file?
|
|
445
|
-
next if macos_junk_entry?(entry.name)
|
|
446
|
-
reject_unsafe_entry!(entry.name)
|
|
447
|
-
dest_path = safe_extract_path(dest_dir, entry.name)
|
|
448
|
-
FileUtils.mkdir_p(File.dirname(dest_path))
|
|
449
|
-
next if File.exist?(dest_path)
|
|
450
|
-
extract_zip_entry(zip_file, entry, dest_dir, entry.name, dest_path)
|
|
451
|
-
end
|
|
452
|
-
end
|
|
453
|
-
end
|
|
454
|
-
|
|
455
|
-
# rubyzip 2.x: extract(entry, absolute_dest_path)
|
|
456
|
-
# rubyzip 3.x: extract(entry, relative_name, destination_directory: dir)
|
|
457
|
-
#
|
|
458
|
-
# Callers are responsible for passing a `dest_path` produced by
|
|
459
|
-
# {#safe_extract_path} so the write can't escape `dest_dir`.
|
|
460
|
-
def extract_zip_entry(zip_file, entry, dest_dir, relative_name, dest_path)
|
|
461
|
-
if zip_file.method(:extract).arity == 2
|
|
462
|
-
zip_file.extract(entry, dest_path)
|
|
463
|
-
else
|
|
464
|
-
zip_file.extract(entry, relative_name, destination_directory: dest_dir)
|
|
465
|
-
end
|
|
466
|
-
end
|
|
467
|
-
|
|
468
|
-
def macos_junk_entry?(name)
|
|
469
|
-
name.start_with?('__MACOSX/') || name.split('/').any? { |part| part == '.DS_Store' || part.start_with?('._') }
|
|
470
|
-
end
|
|
471
|
-
|
|
472
|
-
# Zip Slip preflight — reject entries whose names are obviously unsafe
|
|
473
|
-
# (absolute paths, `..` segments) before we touch the filesystem.
|
|
474
|
-
# {#safe_extract_path} is the final line of defense; this check just
|
|
475
|
-
# fails fast with a clear message. Delegates to {Bulkrax::SafePath} so
|
|
476
|
-
# cloud-file destinations and sources share the same rules.
|
|
477
|
-
#
|
|
478
|
-
# @raise [Bulkrax::UnzipError] if the entry name is unsafe
|
|
479
|
-
<<<<<<< HEAD
|
|
480
|
-
def reject_unsafe_entry!(name)
|
|
481
|
-
return unless name.start_with?('/') || name.split('/').include?('..')
|
|
482
|
-
raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: name)
|
|
483
|
-
=======
|
|
484
|
-
def self.reject_unsafe_entry!(name)
|
|
485
|
-
Bulkrax::SafePath.reject_unsafe_entry!(name)
|
|
486
|
-
rescue Bulkrax::UnsafePathError
|
|
487
|
-
raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: name.dup.force_encoding(Encoding::UTF_8).scrub)
|
|
488
|
-
>>>>>>> acc18e45 (Merge commit from fork)
|
|
489
|
-
end
|
|
490
|
-
|
|
491
|
-
# Zip Slip chokepoint. Resolves `relative_dest` against `dest_dir` and
|
|
492
|
-
# returns the absolute destination path — but only if it stays inside
|
|
493
|
-
# `dest_dir`, including through any symlink. Callers must use this
|
|
494
|
-
# value rather than building their own path with `File.join`, so the
|
|
495
|
-
# path returned is always safe by construction.
|
|
496
|
-
#
|
|
497
|
-
# @return [String] absolute destination path, validated to be inside `dest_dir`
|
|
498
|
-
# @raise [Bulkrax::UnzipError] if `relative_dest` escapes `dest_dir`
|
|
499
|
-
def safe_extract_path(dest_dir, relative_dest)
|
|
500
|
-
Bulkrax::SafePath.confine!(dest_dir, relative_dest)
|
|
501
|
-
rescue Bulkrax::UnsafePathError
|
|
502
|
-
raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: relative_dest)
|
|
503
|
-
end
|
|
504
|
-
|
|
505
|
-
def copy_file(file_to_copy)
|
|
506
|
-
destination = File.join(importer_unzip_path(mkdir: true), File.basename(file_to_copy))
|
|
507
|
-
FileUtils.cp(file_to_copy, destination)
|
|
508
|
-
end
|
|
509
|
-
|
|
510
|
-
def untar(file_to_untar)
|
|
511
|
-
Dir.mkdir(importer_unzip_path(mkdir: true)) unless File.directory?(importer_unzip_path(mkdir: true))
|
|
512
|
-
command = "tar -xzf #{Shellwords.escape(file_to_untar)} -C #{Shellwords.escape(importer_unzip_path)}"
|
|
513
|
-
result = system(command)
|
|
514
|
-
raise "Failed to extract #{file_to_untar}" unless result
|
|
515
|
-
end
|
|
516
|
-
|
|
517
|
-
def zip
|
|
518
|
-
FileUtils.mkdir_p(exporter_export_zip_path)
|
|
519
|
-
|
|
520
|
-
Dir["#{exporter_export_path}/**"].each do |folder|
|
|
521
|
-
zip_path = "#{exporter_export_zip_path.split('/').last}_#{folder.split('/').last}.zip"
|
|
522
|
-
FileUtils.rm_rf("#{exporter_export_zip_path}/#{zip_path}")
|
|
523
|
-
|
|
524
|
-
Zip::File.open(File.join("#{exporter_export_zip_path}/#{zip_path}"), create: true) do |zip_file|
|
|
525
|
-
Dir["#{folder}/**/**"].each do |file|
|
|
526
|
-
zip_file.add(file.sub("#{folder}/", ''), file)
|
|
527
|
-
end
|
|
528
|
-
end
|
|
529
|
-
end
|
|
530
|
-
end
|
|
531
|
-
|
|
532
|
-
# Path for the import
|
|
533
|
-
# @return [String]
|
|
534
|
-
def import_file_path
|
|
535
|
-
@import_file_path ||= real_import_file_path
|
|
536
|
-
end
|
|
537
|
-
|
|
538
|
-
private
|
|
539
|
-
|
|
540
|
-
# @return [String]
|
|
541
|
-
def real_import_file_path
|
|
542
|
-
parser_fields['import_file_path']
|
|
543
|
-
end
|
|
544
|
-
end
|
|
545
|
-
end
|