bulkrax 9.5.1 → 10.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +49 -0
- data/app/controllers/bulkrax/application_controller.rb +83 -0
- data/app/controllers/bulkrax/entries_controller.rb +16 -13
- data/app/controllers/bulkrax/exporters_controller.rb +21 -25
- data/app/controllers/bulkrax/guided_imports_controller.rb +4 -8
- data/app/controllers/bulkrax/importers_controller.rb +84 -46
- data/app/controllers/concerns/bulkrax/datatables_behavior.rb +18 -12
- data/app/controllers/concerns/bulkrax/importer_file_handler.rb +55 -9
- data/app/errors/bulkrax/unsafe_cloud_source_error.rb +11 -0
- data/app/errors/bulkrax/unsafe_path_error.rb +8 -0
- data/app/factories/bulkrax/object_factory.rb +2 -2
- data/app/factories/bulkrax/valkyrie_object_factory.rb +29 -17
- data/app/helpers/bulkrax/validation_helper.rb +1 -1
- data/app/jobs/bulkrax/create_relationships_job.rb +1 -1
- data/app/jobs/bulkrax/download_cloud_file_job.rb +156 -21
- data/app/jobs/bulkrax/import_collection_job.rb +9 -2
- data/app/matchers/bulkrax/application_matcher.rb +5 -2
- data/app/models/bulkrax/csv_entry.rb +1 -1
- data/app/models/bulkrax/current.rb +7 -0
- data/app/models/bulkrax/export_scope.rb +21 -0
- data/app/models/bulkrax/exporter.rb +2 -2
- data/app/models/bulkrax/importer.rb +1 -1
- data/app/models/concerns/bulkrax/ability_behavior.rb +139 -0
- data/app/models/concerns/bulkrax/has_matchers.rb +1 -1
- data/app/models/concerns/bulkrax/import_behavior.rb +3 -3
- data/app/parsers/bulkrax/application_parser.rb +11 -11
- data/app/parsers/bulkrax/application_parser.rb.orig +545 -0
- data/app/parsers/bulkrax/bagit_parser.rb +10 -3
- data/app/parsers/bulkrax/csv_parser.rb +59 -27
- data/app/parsers/bulkrax/oai_dc_parser.rb +1 -1
- data/app/parsers/bulkrax/parser_export_record_set.rb +26 -8
- data/app/parsers/concerns/bulkrax/csv_parser/csv_template_generation.rb +1 -1
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation.rb +1 -2
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_helpers.rb +18 -1
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_hierarchy.rb +2 -2
- data/app/search_builders/bulkrax/export_search_builder.rb +9 -0
- data/app/services/bulkrax/cloud_source_validator.rb +123 -0
- data/app/services/bulkrax/csv_template/column_builder.rb +2 -2
- data/app/services/bulkrax/csv_template/mapping_manager.rb +9 -7
- data/app/services/bulkrax/csv_template/schema_analyzer.rb +1 -5
- data/app/services/bulkrax/safe_path.rb +129 -0
- data/config/locales/bulkrax.en.yml +2 -0
- data/lib/bulkrax/engine.rb +1 -1
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +77 -10
- data/lib/generators/bulkrax/install_generator.rb +22 -10
- data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb +1 -1
- metadata +58 -14
- data/app/factories/bulkrax/valkyrize-hyku.code-workspace +0 -19
- data/lib/wings/custom_queries/find_by_source_identifier.rb +0 -41
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
module Bulkrax
|
|
4
4
|
# rubocop:disable Metrics/ClassLength
|
|
5
5
|
class ImportersController < ::Bulkrax::ApplicationController
|
|
6
|
-
include Hyrax::ThemedLayoutController if
|
|
6
|
+
include Hyrax::ThemedLayoutController if Bulkrax.hyrax_loaded?
|
|
7
7
|
include Bulkrax::DownloadBehavior
|
|
8
8
|
include Bulkrax::API
|
|
9
9
|
include Bulkrax::DatatablesBehavior
|
|
@@ -11,29 +11,39 @@ module Bulkrax
|
|
|
11
11
|
include Bulkrax::ImporterFileHandler
|
|
12
12
|
|
|
13
13
|
protect_from_forgery unless: -> { api_request? }
|
|
14
|
-
|
|
15
|
-
before_action
|
|
16
|
-
|
|
17
|
-
before_action :set_importer, only: [:show, :entry_table, :edit, :update, :destroy, :original_file]
|
|
18
|
-
with_themed_layout 'dashboard' if defined?(::Hyrax)
|
|
14
|
+
# Authentication, record loading and authorization for every action.
|
|
15
|
+
before_action { authenticate_and_authorize!(Bulkrax::Importer, :can_import_works?) }
|
|
16
|
+
with_themed_layout 'dashboard' if Bulkrax.hyrax_loaded?
|
|
19
17
|
|
|
20
18
|
# GET /importers
|
|
21
19
|
def index
|
|
22
20
|
# NOTE: We're paginating this in the browser.
|
|
23
21
|
if api_request?
|
|
22
|
+
# TODO(#1237): Scope API index by token owner once token-to-user wiring
|
|
23
|
+
# is in place. Until then, API clients see all importers
|
|
24
|
+
# to preserve backward-compatibility with existing API consumers.
|
|
24
25
|
@importers = Importer.order(created_at: :desc).all
|
|
25
26
|
json_response('index')
|
|
26
|
-
elsif
|
|
27
|
+
elsif Bulkrax.hyrax_loaded?
|
|
27
28
|
add_importer_breadcrumbs
|
|
28
29
|
end
|
|
29
30
|
end
|
|
30
31
|
|
|
31
32
|
def importer_table
|
|
32
33
|
order = table_order.presence || Arel.sql('last_imported_at DESC NULLS LAST')
|
|
33
|
-
|
|
34
|
+
# TODO(#1237): API requests bypass ownership scoping here too.
|
|
35
|
+
@importers = if api_request?
|
|
36
|
+
Importer.all
|
|
37
|
+
else
|
|
38
|
+
Importer.accessible_by(current_ability)
|
|
39
|
+
end
|
|
34
40
|
@importers = @importers.where(importer_table_search) if importer_table_search.present?
|
|
41
|
+
# Count the filtered relation before applying pagination so the UI receives
|
|
42
|
+
# the total number of matching importers, not just the total number of importers.
|
|
43
|
+
filtered_count = @importers.count
|
|
44
|
+
@importers = @importers.order(order).page(table_page).per(table_per_page)
|
|
35
45
|
respond_to do |format|
|
|
36
|
-
format.json { render json: format_importers(@importers) }
|
|
46
|
+
format.json { render json: format_importers(@importers, filtered_count) }
|
|
37
47
|
end
|
|
38
48
|
end
|
|
39
49
|
|
|
@@ -41,7 +51,7 @@ module Bulkrax
|
|
|
41
51
|
def show
|
|
42
52
|
if api_request?
|
|
43
53
|
json_response('show')
|
|
44
|
-
elsif
|
|
54
|
+
elsif Bulkrax.hyrax_loaded?
|
|
45
55
|
add_importer_breadcrumbs
|
|
46
56
|
add_breadcrumb @importer.name
|
|
47
57
|
end
|
|
@@ -58,10 +68,9 @@ module Bulkrax
|
|
|
58
68
|
|
|
59
69
|
# GET /importers/new
|
|
60
70
|
def new
|
|
61
|
-
@importer = Importer.new
|
|
62
71
|
if api_request?
|
|
63
72
|
json_response('new')
|
|
64
|
-
elsif
|
|
73
|
+
elsif Bulkrax.hyrax_loaded?
|
|
65
74
|
add_importer_breadcrumbs
|
|
66
75
|
add_breadcrumb t(:'bulkrax.headings.new_importer')
|
|
67
76
|
end
|
|
@@ -81,7 +90,7 @@ module Bulkrax
|
|
|
81
90
|
def edit
|
|
82
91
|
if api_request?
|
|
83
92
|
json_response('edit')
|
|
84
|
-
elsif
|
|
93
|
+
elsif Bulkrax.hyrax_loaded?
|
|
85
94
|
add_importer_breadcrumbs
|
|
86
95
|
add_breadcrumb @importer.name, bulkrax.importer_path(@importer.id)
|
|
87
96
|
add_breadcrumb t(:'bulkrax.headings.edit_importer')
|
|
@@ -92,22 +101,30 @@ module Bulkrax
|
|
|
92
101
|
# rubocop:disable Metrics/MethodLength
|
|
93
102
|
# rubocop:disable Metrics/AbcSize
|
|
94
103
|
def create
|
|
95
|
-
# rubocop:disable Style/IfInsideElse
|
|
96
104
|
if api_request?
|
|
97
105
|
return return_json_response unless valid_create_params?
|
|
106
|
+
# authenticate_and_authorize! doesn't build for API requests; build it here
|
|
107
|
+
@importer ||= Importer.new(importer_params)
|
|
98
108
|
end
|
|
109
|
+
@importer.user_id = current_user.id
|
|
99
110
|
uploads = uploaded_files_scope
|
|
100
111
|
file = file_param
|
|
101
112
|
cloud_files = cloud_params
|
|
102
113
|
|
|
103
|
-
@importer = Importer.new(importer_params)
|
|
104
114
|
field_mapping_params
|
|
105
115
|
@importer.validate_only = true if params[:commit] == 'Create and Validate'
|
|
106
116
|
# the following line is needed to handle updating remote files of a FileSet
|
|
107
117
|
# on a new import otherwise it only gets updated during the update path
|
|
108
118
|
@importer.parser_fields['update_files'] = true if params[:commit] == 'Create and Import'
|
|
119
|
+
|
|
120
|
+
# Reject an unsafe cloud selection before the importer is ever
|
|
121
|
+
# persisted, rather than after save when a row would already exist.
|
|
122
|
+
preflight_cloud_files!(cloud_files)
|
|
123
|
+
|
|
109
124
|
if @importer.save
|
|
110
|
-
files_for_import(file, cloud_files, uploads)
|
|
125
|
+
return render_create_failure if files_for_import(file, cloud_files, uploads) && !@importer.save
|
|
126
|
+
dispatch_cloud_downloads!
|
|
127
|
+
|
|
111
128
|
if params[:commit] == 'Create and Import'
|
|
112
129
|
Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id)
|
|
113
130
|
render_request('Importer was successfully created and import has been queued.')
|
|
@@ -118,13 +135,10 @@ module Bulkrax
|
|
|
118
135
|
render_request('Importer was successfully created.')
|
|
119
136
|
end
|
|
120
137
|
else
|
|
121
|
-
|
|
122
|
-
json_response('create', :unprocessable_entity)
|
|
123
|
-
else
|
|
124
|
-
render :new
|
|
125
|
-
end
|
|
138
|
+
render_create_failure
|
|
126
139
|
end
|
|
127
|
-
|
|
140
|
+
rescue Bulkrax::UnsafeCloudSourceError, Bulkrax::UnsafePathError => e
|
|
141
|
+
reject_unsafe_cloud_files(e)
|
|
128
142
|
end
|
|
129
143
|
# rubocop:enable Metrics/AbcSize
|
|
130
144
|
|
|
@@ -142,11 +156,19 @@ module Bulkrax
|
|
|
142
156
|
# Skipped during a continue
|
|
143
157
|
field_mapping_params if params[:importer][:parser_fields].present?
|
|
144
158
|
|
|
145
|
-
|
|
146
|
-
|
|
159
|
+
# Assign the proposed attributes in memory first, so an unsafe cloud
|
|
160
|
+
# selection is rejected before anything is persisted; the rescue
|
|
161
|
+
# below then leaves the importer's persisted attributes untouched.
|
|
162
|
+
@importer.assign_attributes(importer_params)
|
|
163
|
+
preflight_cloud_files!(cloud_files)
|
|
164
|
+
|
|
165
|
+
if @importer.save
|
|
166
|
+
return render_update_failure if files_for_import(file, cloud_files, uploads) && !@importer.save
|
|
167
|
+
return render_update_failure if params[:commit] != 'Update Importer' && !set_files_parser_fields
|
|
168
|
+
dispatch_cloud_downloads!
|
|
169
|
+
|
|
147
170
|
# do not perform the import
|
|
148
171
|
unless params[:commit] == 'Update Importer'
|
|
149
|
-
set_files_parser_fields
|
|
150
172
|
Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id, update_harvest)
|
|
151
173
|
end
|
|
152
174
|
if api_request?
|
|
@@ -155,12 +177,10 @@ module Bulkrax
|
|
|
155
177
|
redirect_to importers_path, notice: 'Importer was successfully updated.'
|
|
156
178
|
end
|
|
157
179
|
else
|
|
158
|
-
|
|
159
|
-
json_response('update', :unprocessable_entity, 'Something went wrong.')
|
|
160
|
-
else
|
|
161
|
-
render :edit
|
|
162
|
-
end
|
|
180
|
+
render_update_failure
|
|
163
181
|
end
|
|
182
|
+
rescue Bulkrax::UnsafeCloudSourceError, Bulkrax::UnsafePathError => e
|
|
183
|
+
reject_unsafe_cloud_files(e)
|
|
164
184
|
end
|
|
165
185
|
# rubocop:enable all
|
|
166
186
|
# rubocop:enable Metrics/MethodLength
|
|
@@ -177,7 +197,6 @@ module Bulkrax
|
|
|
177
197
|
|
|
178
198
|
# PUT /importers/1
|
|
179
199
|
def continue
|
|
180
|
-
@importer = Importer.find(params[:importer_id])
|
|
181
200
|
params[:importer] = { name: @importer.name }
|
|
182
201
|
@importer.validate_only = false
|
|
183
202
|
update
|
|
@@ -185,8 +204,7 @@ module Bulkrax
|
|
|
185
204
|
|
|
186
205
|
# GET /importer/1/upload_corrected_entries
|
|
187
206
|
def upload_corrected_entries
|
|
188
|
-
|
|
189
|
-
return unless defined?(::Hyrax)
|
|
207
|
+
return unless Bulkrax.hyrax_loaded?
|
|
190
208
|
add_breadcrumb t(:'hyrax.controls.home'), main_app.root_path
|
|
191
209
|
add_breadcrumb t(:'hyrax.dashboard.breadcrumbs.admin'), hyrax.dashboard_path
|
|
192
210
|
add_breadcrumb t(:'bulkrax.headings.importers'), bulkrax.importers_path
|
|
@@ -197,7 +215,6 @@ module Bulkrax
|
|
|
197
215
|
# POST /importer/1/upload_corrected_entries_file
|
|
198
216
|
def upload_corrected_entries_file
|
|
199
217
|
file = params[:importer][:parser_fields].delete(:file)
|
|
200
|
-
@importer = Importer.find(params[:importer_id])
|
|
201
218
|
if file.present?
|
|
202
219
|
@importer[:parser_fields]['partial_import_file_path'] = @importer.parser.write_partial_import_file(file)
|
|
203
220
|
@importer.save
|
|
@@ -242,18 +259,12 @@ module Bulkrax
|
|
|
242
259
|
|
|
243
260
|
# GET /importers/1/export_errors
|
|
244
261
|
def export_errors
|
|
245
|
-
@importer = Importer.find(params[:importer_id])
|
|
246
262
|
@importer.write_errored_entries_file
|
|
247
263
|
send_content
|
|
248
264
|
end
|
|
249
265
|
|
|
250
266
|
private
|
|
251
267
|
|
|
252
|
-
# Use callbacks to share common setup or constraints between actions.
|
|
253
|
-
def set_importer
|
|
254
|
-
@importer = Importer.find(params[:id] || params[:importer_id])
|
|
255
|
-
end
|
|
256
|
-
|
|
257
268
|
def importable_params
|
|
258
269
|
params.except(:selected_files)
|
|
259
270
|
end
|
|
@@ -267,7 +278,6 @@ module Bulkrax
|
|
|
267
278
|
importable_params.require(:importer).permit(
|
|
268
279
|
:name,
|
|
269
280
|
:admin_set_id,
|
|
270
|
-
:user_id,
|
|
271
281
|
:frequency,
|
|
272
282
|
:parser_klass,
|
|
273
283
|
:limit,
|
|
@@ -332,6 +342,36 @@ module Bulkrax
|
|
|
332
342
|
'text/csv'
|
|
333
343
|
end
|
|
334
344
|
|
|
345
|
+
# Reports an unsafe cloud-file batch without leaking the error's class
|
|
346
|
+
# or backtrace (which could include a provider auth header) to the
|
|
347
|
+
# response; `e.message` is always one of our own static, credential-free
|
|
348
|
+
# strings (see Bulkrax::CloudSourceValidator / Bulkrax::SafePath).
|
|
349
|
+
def reject_unsafe_cloud_files(e)
|
|
350
|
+
Rails.logger.error("Bulkrax: rejected unsafe cloud file batch for importer #{@importer&.id}: #{e.class}: #{e.message}")
|
|
351
|
+
if api_request?
|
|
352
|
+
json_response('create', :unprocessable_entity, e.message)
|
|
353
|
+
else
|
|
354
|
+
flash[:error] = e.message
|
|
355
|
+
redirect_back fallback_location: bulkrax.importers_path
|
|
356
|
+
end
|
|
357
|
+
end
|
|
358
|
+
|
|
359
|
+
def render_create_failure
|
|
360
|
+
if api_request?
|
|
361
|
+
json_response('create', :unprocessable_entity)
|
|
362
|
+
else
|
|
363
|
+
render :new
|
|
364
|
+
end
|
|
365
|
+
end
|
|
366
|
+
|
|
367
|
+
def render_update_failure
|
|
368
|
+
if api_request?
|
|
369
|
+
json_response('update', :unprocessable_entity, 'Something went wrong.')
|
|
370
|
+
else
|
|
371
|
+
render :edit
|
|
372
|
+
end
|
|
373
|
+
end
|
|
374
|
+
|
|
335
375
|
def render_request(message, validate_only = false)
|
|
336
376
|
if api_request?
|
|
337
377
|
json_response('create', :created, message)
|
|
@@ -347,6 +387,8 @@ module Bulkrax
|
|
|
347
387
|
params[:commit] == 'Update and Harvest Updated Items'
|
|
348
388
|
end
|
|
349
389
|
|
|
390
|
+
# @return [Boolean] the result of the save, so the caller can treat a
|
|
391
|
+
# failure here like an ordinary validation failure
|
|
350
392
|
def set_files_parser_fields
|
|
351
393
|
@importer.parser_fields['update_files'] =
|
|
352
394
|
@importer.parser_fields['replace_files'] =
|
|
@@ -359,16 +401,12 @@ module Bulkrax
|
|
|
359
401
|
elsif params[:commit] == 'Remove and Rerun'
|
|
360
402
|
@importer.parser_fields['remove_and_rerun'] = true
|
|
361
403
|
elsif params[:commit] == 'Update and Harvest Updated Items'
|
|
362
|
-
return
|
|
404
|
+
return true
|
|
363
405
|
else
|
|
364
406
|
@importer.parser_fields['metadata_only'] = true
|
|
365
407
|
end
|
|
366
408
|
@importer.save
|
|
367
409
|
end
|
|
368
|
-
|
|
369
|
-
def check_permissions
|
|
370
|
-
raise CanCan::AccessDenied unless current_ability.can_import_works?
|
|
371
|
-
end
|
|
372
410
|
end
|
|
373
411
|
# rubocop:enable Metrics/ClassLength
|
|
374
412
|
end
|
|
@@ -83,7 +83,7 @@ module Bulkrax
|
|
|
83
83
|
@exporter_table_search
|
|
84
84
|
end
|
|
85
85
|
|
|
86
|
-
def format_importers(importers)
|
|
86
|
+
def format_importers(importers, filtered_count = Bulkrax::Importer.accessible_by(current_ability).count)
|
|
87
87
|
result = importers.map do |i|
|
|
88
88
|
{
|
|
89
89
|
name: view_context.link_to(i.name, view_context.importer_path(i)),
|
|
@@ -102,12 +102,12 @@ module Bulkrax
|
|
|
102
102
|
end
|
|
103
103
|
{
|
|
104
104
|
data: result,
|
|
105
|
-
recordsTotal: Bulkrax::Importer.count,
|
|
106
|
-
recordsFiltered:
|
|
105
|
+
recordsTotal: api_request? ? Bulkrax::Importer.count : Bulkrax::Importer.accessible_by(current_ability).count,
|
|
106
|
+
recordsFiltered: filtered_count
|
|
107
107
|
}
|
|
108
108
|
end
|
|
109
109
|
|
|
110
|
-
def format_exporters(exporters)
|
|
110
|
+
def format_exporters(exporters, filtered_count = Bulkrax::Exporter.accessible_by(current_ability).count)
|
|
111
111
|
result = exporters.map do |e|
|
|
112
112
|
{
|
|
113
113
|
name: view_context.link_to(e.name, view_context.exporter_path(e)),
|
|
@@ -119,8 +119,8 @@ module Bulkrax
|
|
|
119
119
|
end
|
|
120
120
|
{
|
|
121
121
|
data: result,
|
|
122
|
-
recordsTotal: Bulkrax::Exporter.count,
|
|
123
|
-
recordsFiltered:
|
|
122
|
+
recordsTotal: Bulkrax::Exporter.accessible_by(current_ability).count,
|
|
123
|
+
recordsFiltered: filtered_count
|
|
124
124
|
}
|
|
125
125
|
end
|
|
126
126
|
|
|
@@ -146,8 +146,10 @@ module Bulkrax
|
|
|
146
146
|
def entry_util_links(e, item)
|
|
147
147
|
links = []
|
|
148
148
|
links << view_context.link_to(view_context.raw('<span class="fa fa-info-circle"></span>'), view_context.item_entry_path(item, e))
|
|
149
|
-
|
|
150
|
-
|
|
149
|
+
if can?(:update, item)
|
|
150
|
+
links << "<a class='fa fa-repeat' data-toggle='modal' data-target='#bulkraxItemModal' data-entry-id='#{e.id}'></a>" if view_context.an_importer?(item)
|
|
151
|
+
links << view_context.link_to(view_context.raw('<span class="fa fa-trash"></span>'), view_context.item_entry_path(item, e), method: :delete, data: { confirm: 'This will delete the entry and any work associated with it. Are you sure?' })
|
|
152
|
+
end
|
|
151
153
|
links.join(" ")
|
|
152
154
|
end
|
|
153
155
|
|
|
@@ -166,16 +168,20 @@ module Bulkrax
|
|
|
166
168
|
def importer_util_links(i)
|
|
167
169
|
links = []
|
|
168
170
|
links << view_context.link_to(view_context.raw('<span class="fa fa-info-circle"></span>'), importer_path(i))
|
|
169
|
-
|
|
170
|
-
|
|
171
|
+
if can?(:update, i)
|
|
172
|
+
links << view_context.link_to(view_context.raw('<span class="fa fa-pencil"></span>'), edit_importer_path(i))
|
|
173
|
+
links << view_context.link_to(view_context.raw('<span class="fa fa-remove"></span>'), i, method: :delete, data: { confirm: 'Are you sure?' })
|
|
174
|
+
end
|
|
171
175
|
links.join(" ")
|
|
172
176
|
end
|
|
173
177
|
|
|
174
178
|
def exporter_util_links(i)
|
|
175
179
|
links = []
|
|
176
180
|
links << view_context.link_to(view_context.raw('<span class="fa fa-info-circle"></span>'), exporter_path(i))
|
|
177
|
-
|
|
178
|
-
|
|
181
|
+
if can?(:update, i)
|
|
182
|
+
links << view_context.link_to(view_context.raw('<span class="fa fa-pencil"></span>'), edit_exporter_path(i), data: { turbolinks: false })
|
|
183
|
+
links << view_context.link_to(view_context.raw('<span class="fa fa-remove"></span>'), i, method: :delete, data: { confirm: 'Are you sure?' })
|
|
184
|
+
end
|
|
179
185
|
links.join(" ")
|
|
180
186
|
end
|
|
181
187
|
|
|
@@ -43,7 +43,7 @@ module Bulkrax
|
|
|
43
43
|
end
|
|
44
44
|
|
|
45
45
|
def uploaded_files_scope
|
|
46
|
-
return [] unless
|
|
46
|
+
return [] unless Bulkrax.hyrax_loaded?
|
|
47
47
|
|
|
48
48
|
base = Hyrax::UploadedFile.where(id: params[:uploaded_files])
|
|
49
49
|
if respond_to?(:current_user) && current_user.present?
|
|
@@ -139,21 +139,38 @@ module Bulkrax
|
|
|
139
139
|
csv_by_depth[shallowest_depth]
|
|
140
140
|
end
|
|
141
141
|
|
|
142
|
-
#
|
|
142
|
+
# Validates every selected cloud-file entry — raw name and source
|
|
143
|
+
# safety — before the importer is created or updated at all. Must not
|
|
144
|
+
# touch the importer's workspace (no path_for_import, no directory
|
|
145
|
+
# creation, no download): for #create that workspace doesn't exist
|
|
146
|
+
# yet, and for #update it must not be assumed safe to touch before the
|
|
147
|
+
# proposed attributes are known to be acceptable.
|
|
148
|
+
#
|
|
149
|
+
# @param cloud_files [Hash, nil] cloud file specs from browse-everything
|
|
150
|
+
# @raise [Bulkrax::UnsafePathError, Bulkrax::UnsafeCloudSourceError] on the first unsafe entry
|
|
151
|
+
def preflight_cloud_files!(cloud_files)
|
|
152
|
+
return if cloud_files.blank?
|
|
153
|
+
|
|
154
|
+
cloud_files.values.each { |file| Bulkrax::CloudSourceValidator.preflight!(file) }
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Assigns files/cloud selections onto the importer record's in-memory
|
|
158
|
+
# parser fields, but does not save: the caller (ImportersController)
|
|
159
|
+
# owns the second save, so it can check its result and avoid rendering
|
|
160
|
+
# success or enqueueing an import on failure.
|
|
161
|
+
#
|
|
143
162
|
# @param file [ActionDispatch::Http::UploadedFile, nil] a directly uploaded file
|
|
144
163
|
# @param cloud_files [Hash, nil] cloud file paths from browse-everything
|
|
145
164
|
# @param uploads [ActiveRecord::Relation, Array, nil] Hyrax::UploadedFile records
|
|
165
|
+
# @return [Boolean] true if any parser field was touched and a second save is needed
|
|
146
166
|
def files_for_import(file, cloud_files, uploads)
|
|
147
|
-
|
|
167
|
+
@cloud_download_plan = nil
|
|
168
|
+
return false if file.blank? && cloud_files.blank? && uploads.blank?
|
|
148
169
|
|
|
149
170
|
@importer[:parser_fields]['import_file_path'] = @importer.parser.write_import_file(file) if file.present?
|
|
150
171
|
if cloud_files.present?
|
|
151
172
|
@importer[:parser_fields]['cloud_file_paths'] = cloud_files
|
|
152
|
-
|
|
153
|
-
# For CSV, we expect only file uploads, so we won't get the file_path back
|
|
154
|
-
# and we expect the import_file_path to be set already
|
|
155
|
-
target = @importer.parser.retrieve_cloud_files(cloud_files, @importer)
|
|
156
|
-
@importer[:parser_fields]['import_file_path'] = target if target.present?
|
|
173
|
+
assign_cloud_file_paths(cloud_files)
|
|
157
174
|
end
|
|
158
175
|
|
|
159
176
|
if uploads.present?
|
|
@@ -162,7 +179,36 @@ module Bulkrax
|
|
|
162
179
|
end
|
|
163
180
|
end
|
|
164
181
|
|
|
165
|
-
|
|
182
|
+
true
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# For a parser that separates planning from dispatch (currently only
|
|
186
|
+
# CsvParser), resolves and validates the whole batch and records the
|
|
187
|
+
# target paths, deferring enqueue until {#dispatch_cloud_downloads!} is
|
|
188
|
+
# called after the controller persists those paths. Other parsers
|
|
189
|
+
# (e.g. BagitParser) already complete cloud retrieval synchronously
|
|
190
|
+
# and return the resolved path directly, so there's nothing left to
|
|
191
|
+
# dispatch.
|
|
192
|
+
def assign_cloud_file_paths(cloud_files)
|
|
193
|
+
parser = @importer.parser
|
|
194
|
+
if parser.respond_to?(:plan_cloud_files)
|
|
195
|
+
@cloud_download_plan = parser.plan_cloud_files(cloud_files)
|
|
196
|
+
@importer[:parser_fields]['original_file_paths'] =
|
|
197
|
+
@cloud_download_plan.map { |_file, _relative_filename, target_file| target_file }
|
|
198
|
+
else
|
|
199
|
+
target = parser.retrieve_cloud_files(cloud_files, @importer)
|
|
200
|
+
@importer[:parser_fields]['import_file_path'] = target if target.present?
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# Dispatches a plan prepared by {#assign_cloud_file_paths}, if any.
|
|
205
|
+
# Called by the controller only after it has successfully persisted
|
|
206
|
+
# the resolved paths, so a download is never enqueued for a path the
|
|
207
|
+
# importer doesn't actually have on record.
|
|
208
|
+
def dispatch_cloud_downloads!
|
|
209
|
+
return if @cloud_download_plan.blank?
|
|
210
|
+
|
|
211
|
+
@importer.parser.dispatch_cloud_files!(@cloud_download_plan, @importer)
|
|
166
212
|
end
|
|
167
213
|
|
|
168
214
|
def write_files(files)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
# Raised by {Bulkrax::CloudSourceValidator} when a cloud-file source
|
|
5
|
+
# (the `url` a user selected via browse-everything, or supplied directly
|
|
6
|
+
# in importer params) can't be safely downloaded: a scheme other than
|
|
7
|
+
# `file`/`http`/`https`, a `file:` source outside the configured
|
|
8
|
+
# filesystem-provider root, or an HTTP(S) source that resolves to a
|
|
9
|
+
# private, loopback, or link-local/metadata address.
|
|
10
|
+
class UnsafeCloudSourceError < StandardError; end
|
|
11
|
+
end
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
# Raised by {Bulkrax::SafePath} when a path would escape its configured
|
|
5
|
+
# root, whether that root is a ZIP extraction directory, an importer's
|
|
6
|
+
# trusted workspace, or a configured filesystem-provider home.
|
|
7
|
+
class UnsafePathError < StandardError; end
|
|
8
|
+
end
|
|
@@ -168,8 +168,8 @@ module Bulkrax
|
|
|
168
168
|
end
|
|
169
169
|
|
|
170
170
|
def self.solr_name(field_name)
|
|
171
|
-
if defined?(Hyrax)
|
|
172
|
-
Hyrax.index_field_mapper.solr_name(field_name)
|
|
171
|
+
if defined?(Hyrax) && Hyrax.respond_to?(:config) && Hyrax.config.respond_to?(:index_field_mapper)
|
|
172
|
+
Hyrax.config.index_field_mapper.solr_name(field_name)
|
|
173
173
|
else
|
|
174
174
|
ActiveFedora.index_field_mapper.solr_name(field_name)
|
|
175
175
|
end
|
|
@@ -280,11 +280,14 @@ module Bulkrax
|
|
|
280
280
|
# @param admin_set_id [String, nil] admin set used to resolve contexts
|
|
281
281
|
# @return [Array<String>]
|
|
282
282
|
def self.schema_properties(klass:, admin_set_id: nil)
|
|
283
|
-
|
|
283
|
+
entry = schema_cache_entry(klass: klass, admin_set_id: admin_set_id)
|
|
284
|
+
entry[:properties] ||= entry[:schema].map { |k| k.name.to_s }.freeze
|
|
284
285
|
end
|
|
285
286
|
|
|
286
287
|
##
|
|
287
|
-
# Returns the schema for a model, memoized per (klass, admin_set_id)
|
|
288
|
+
# Returns the schema for a model, memoized per (tenant, klass, admin_set_id)
|
|
289
|
+
# for the current request or job only, so a new metadata profile or a
|
|
290
|
+
# change to an admin set's contexts is picked up without a restart.
|
|
288
291
|
# Delegates to +Hyrax.schema_for+ when available so that context-gated
|
|
289
292
|
# properties are included without Bulkrax knowing about flexibility internals.
|
|
290
293
|
#
|
|
@@ -292,16 +295,25 @@ module Bulkrax
|
|
|
292
295
|
# @param admin_set_id [String, nil]
|
|
293
296
|
# @return [Dry::Types::Hash]
|
|
294
297
|
def self.cached_schema_for(klass:, admin_set_id: nil)
|
|
295
|
-
|
|
296
|
-
key = [klass.name, admin_set_id].compact.join('|')
|
|
297
|
-
@cached_schema_map[key] ||=
|
|
298
|
-
if admin_set_id.present? && defined?(Hyrax) && Hyrax.respond_to?(:schema_for)
|
|
299
|
-
Hyrax.schema_for(klass: klass, admin_set_id: admin_set_id)
|
|
300
|
-
else
|
|
301
|
-
klass.new.singleton_class.schema || klass.schema
|
|
302
|
-
end
|
|
298
|
+
schema_cache_entry(klass: klass, admin_set_id: admin_set_id)[:schema]
|
|
303
299
|
end
|
|
304
300
|
|
|
301
|
+
def self.schema_cache_entry(klass:, admin_set_id:)
|
|
302
|
+
tenant = defined?(Apartment::Tenant) ? Apartment::Tenant.current : 'single'
|
|
303
|
+
Bulkrax::Current.schema_cache ||= {}
|
|
304
|
+
Bulkrax::Current.schema_cache[[tenant, klass, admin_set_id]] ||= { schema: resolve_schema(klass: klass, admin_set_id: admin_set_id) }
|
|
305
|
+
end
|
|
306
|
+
private_class_method :schema_cache_entry
|
|
307
|
+
|
|
308
|
+
def self.resolve_schema(klass:, admin_set_id:)
|
|
309
|
+
if admin_set_id.present? && defined?(Hyrax) && Hyrax.respond_to?(:schema_for)
|
|
310
|
+
Hyrax.schema_for(klass: klass, admin_set_id: admin_set_id)
|
|
311
|
+
else
|
|
312
|
+
klass.new.singleton_class.schema || klass.schema
|
|
313
|
+
end
|
|
314
|
+
end
|
|
315
|
+
private_class_method :resolve_schema
|
|
316
|
+
|
|
305
317
|
def self.ordered_file_sets_for(object)
|
|
306
318
|
return [] if object.blank?
|
|
307
319
|
|
|
@@ -434,7 +446,7 @@ module Bulkrax
|
|
|
434
446
|
|
|
435
447
|
file_attrs = []
|
|
436
448
|
uploads.each_with_index do |f, index|
|
|
437
|
-
file_attrs <<
|
|
449
|
+
file_attrs << { uploaded_file_id: f["id"].to_s, filename: files[index]["file_name"] }.merge(additional_attributes[index])
|
|
438
450
|
end
|
|
439
451
|
file_attrs.compact.uniq
|
|
440
452
|
end
|
|
@@ -501,7 +513,7 @@ module Bulkrax
|
|
|
501
513
|
@permitted_attributes ||= begin
|
|
502
514
|
bare = base_permitted_attributes + if klass.respond_to?(:schema)
|
|
503
515
|
admin_set_id = attributes[:admin_set_id] || attributes['admin_set_id']
|
|
504
|
-
Bulkrax::ValkyrieObjectFactory.schema_properties(klass: klass, admin_set_id: admin_set_id)
|
|
516
|
+
Bulkrax::ValkyrieObjectFactory.schema_properties(klass: klass, admin_set_id: admin_set_id).map(&:to_sym)
|
|
505
517
|
else
|
|
506
518
|
klass.properties.keys.map(&:to_sym)
|
|
507
519
|
end
|
|
@@ -590,18 +602,18 @@ module Bulkrax
|
|
|
590
602
|
|
|
591
603
|
# @Override Destroy existing files with Hyrax::Transactions
|
|
592
604
|
def destroy_existing_files(object: @object)
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
existing_files.each do |fs|
|
|
605
|
+
destroyed_ids = []
|
|
606
|
+
Hyrax.custom_queries.find_child_file_sets(resource: object).each do |fs|
|
|
597
607
|
transactions["file_set.destroy"]
|
|
598
608
|
.with_step_args("file_set.remove_from_work" => { user: @user },
|
|
599
609
|
"file_set.delete" => { user: @user })
|
|
600
610
|
.call(fs)
|
|
601
611
|
.value!
|
|
612
|
+
destroyed_ids << fs.id
|
|
602
613
|
end
|
|
614
|
+
return if destroyed_ids.empty?
|
|
603
615
|
|
|
604
|
-
object.member_ids
|
|
616
|
+
object.member_ids -= destroyed_ids
|
|
605
617
|
object.rendering_ids = []
|
|
606
618
|
object.representative_id = nil
|
|
607
619
|
object.thumbnail_id = nil
|
|
@@ -114,7 +114,7 @@ module Bulkrax
|
|
|
114
114
|
# We can use Hyrax's lock manager when we have one available.
|
|
115
115
|
# However it's not certain that this is actually working, so to be
|
|
116
116
|
# as safe as possible, we will reload resources before we update.
|
|
117
|
-
if
|
|
117
|
+
if Bulkrax.hyrax_loaded?
|
|
118
118
|
include Hyrax::Lockable
|
|
119
119
|
|
|
120
120
|
def conditionally_acquire_lock_for(*args, &block)
|