bulkrax 9.5.2 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +49 -0
- data/app/controllers/bulkrax/application_controller.rb +83 -0
- data/app/controllers/bulkrax/entries_controller.rb +16 -13
- data/app/controllers/bulkrax/exporters_controller.rb +21 -25
- data/app/controllers/bulkrax/guided_imports_controller.rb +4 -8
- data/app/controllers/bulkrax/importers_controller.rb +43 -87
- data/app/controllers/concerns/bulkrax/datatables_behavior.rb +18 -12
- data/app/controllers/concerns/bulkrax/importer_file_handler.rb +9 -55
- data/app/factories/bulkrax/object_factory.rb +2 -2
- data/app/factories/bulkrax/valkyrie_object_factory.rb +29 -17
- data/app/helpers/bulkrax/validation_helper.rb +1 -1
- data/app/jobs/bulkrax/create_relationships_job.rb +1 -1
- data/app/jobs/bulkrax/download_cloud_file_job.rb +21 -156
- data/app/jobs/bulkrax/import_collection_job.rb +9 -2
- data/app/matchers/bulkrax/application_matcher.rb +5 -2
- data/app/models/bulkrax/csv_entry.rb +1 -1
- data/app/models/bulkrax/current.rb +7 -0
- data/app/models/bulkrax/export_scope.rb +21 -0
- data/app/models/bulkrax/exporter.rb +2 -2
- data/app/models/bulkrax/importer.rb +1 -1
- data/app/models/concerns/bulkrax/ability_behavior.rb +139 -0
- data/app/models/concerns/bulkrax/has_matchers.rb +1 -1
- data/app/models/concerns/bulkrax/import_behavior.rb +3 -3
- data/app/parsers/bulkrax/application_parser.rb +10 -10
- data/app/parsers/bulkrax/bagit_parser.rb +3 -10
- data/app/parsers/bulkrax/csv_parser.rb +28 -58
- data/app/parsers/bulkrax/oai_dc_parser.rb +1 -1
- data/app/parsers/bulkrax/parser_export_record_set.rb +26 -8
- data/app/parsers/concerns/bulkrax/csv_parser/csv_template_generation.rb +1 -1
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation.rb +1 -2
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_helpers.rb +18 -1
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_hierarchy.rb +2 -2
- data/app/search_builders/bulkrax/export_search_builder.rb +9 -0
- data/app/services/bulkrax/csv_template/column_builder.rb +2 -2
- data/app/services/bulkrax/csv_template/mapping_manager.rb +9 -7
- data/app/services/bulkrax/csv_template/schema_analyzer.rb +1 -5
- data/config/locales/bulkrax.en.yml +0 -2
- data/lib/bulkrax/engine.rb +1 -1
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +76 -11
- data/lib/generators/bulkrax/install_generator.rb +22 -10
- data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb +1 -1
- metadata +32 -34
- data/app/errors/bulkrax/unsafe_cloud_source_error.rb +0 -11
- data/app/errors/bulkrax/unsafe_path_error.rb +0 -8
- data/app/parsers/bulkrax/application_parser.rb.orig +0 -545
- data/app/services/bulkrax/cloud_source_validator.rb +0 -123
- data/app/services/bulkrax/safe_path.rb +0 -129
- data/lib/wings/custom_queries/find_by_source_identifier.rb +0 -41
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
module Bulkrax
|
|
4
4
|
# rubocop:disable Metrics/ClassLength
|
|
5
5
|
class ImportersController < ::Bulkrax::ApplicationController
|
|
6
|
-
include Hyrax::ThemedLayoutController if
|
|
6
|
+
include Hyrax::ThemedLayoutController if Bulkrax.hyrax_loaded?
|
|
7
7
|
include Bulkrax::DownloadBehavior
|
|
8
8
|
include Bulkrax::API
|
|
9
9
|
include Bulkrax::DatatablesBehavior
|
|
@@ -11,29 +11,39 @@ module Bulkrax
|
|
|
11
11
|
include Bulkrax::ImporterFileHandler
|
|
12
12
|
|
|
13
13
|
protect_from_forgery unless: -> { api_request? }
|
|
14
|
-
|
|
15
|
-
before_action
|
|
16
|
-
|
|
17
|
-
before_action :set_importer, only: [:show, :entry_table, :edit, :update, :destroy, :original_file]
|
|
18
|
-
with_themed_layout 'dashboard' if defined?(::Hyrax)
|
|
14
|
+
# Authentication, record loading and authorization for every action.
|
|
15
|
+
before_action { authenticate_and_authorize!(Bulkrax::Importer, :can_import_works?) }
|
|
16
|
+
with_themed_layout 'dashboard' if Bulkrax.hyrax_loaded?
|
|
19
17
|
|
|
20
18
|
# GET /importers
|
|
21
19
|
def index
|
|
22
20
|
# NOTE: We're paginating this in the browser.
|
|
23
21
|
if api_request?
|
|
22
|
+
# TODO(#1237): Scope API index by token owner once token-to-user wiring
|
|
23
|
+
# is in place. Until then, API clients see all importers
|
|
24
|
+
# to preserve backward-compatibility with existing API consumers.
|
|
24
25
|
@importers = Importer.order(created_at: :desc).all
|
|
25
26
|
json_response('index')
|
|
26
|
-
elsif
|
|
27
|
+
elsif Bulkrax.hyrax_loaded?
|
|
27
28
|
add_importer_breadcrumbs
|
|
28
29
|
end
|
|
29
30
|
end
|
|
30
31
|
|
|
31
32
|
def importer_table
|
|
32
33
|
order = table_order.presence || Arel.sql('last_imported_at DESC NULLS LAST')
|
|
33
|
-
|
|
34
|
+
# TODO(#1237): API requests bypass ownership scoping here too.
|
|
35
|
+
@importers = if api_request?
|
|
36
|
+
Importer.all
|
|
37
|
+
else
|
|
38
|
+
Importer.accessible_by(current_ability)
|
|
39
|
+
end
|
|
34
40
|
@importers = @importers.where(importer_table_search) if importer_table_search.present?
|
|
41
|
+
# Count the filtered relation before applying pagination so the UI receives
|
|
42
|
+
# the total number of matching importers, not just the total number of importers.
|
|
43
|
+
filtered_count = @importers.count
|
|
44
|
+
@importers = @importers.order(order).page(table_page).per(table_per_page)
|
|
35
45
|
respond_to do |format|
|
|
36
|
-
format.json { render json: format_importers(@importers) }
|
|
46
|
+
format.json { render json: format_importers(@importers, filtered_count) }
|
|
37
47
|
end
|
|
38
48
|
end
|
|
39
49
|
|
|
@@ -41,7 +51,7 @@ module Bulkrax
|
|
|
41
51
|
def show
|
|
42
52
|
if api_request?
|
|
43
53
|
json_response('show')
|
|
44
|
-
elsif
|
|
54
|
+
elsif Bulkrax.hyrax_loaded?
|
|
45
55
|
add_importer_breadcrumbs
|
|
46
56
|
add_breadcrumb @importer.name
|
|
47
57
|
end
|
|
@@ -58,10 +68,9 @@ module Bulkrax
|
|
|
58
68
|
|
|
59
69
|
# GET /importers/new
|
|
60
70
|
def new
|
|
61
|
-
@importer = Importer.new
|
|
62
71
|
if api_request?
|
|
63
72
|
json_response('new')
|
|
64
|
-
elsif
|
|
73
|
+
elsif Bulkrax.hyrax_loaded?
|
|
65
74
|
add_importer_breadcrumbs
|
|
66
75
|
add_breadcrumb t(:'bulkrax.headings.new_importer')
|
|
67
76
|
end
|
|
@@ -81,7 +90,7 @@ module Bulkrax
|
|
|
81
90
|
def edit
|
|
82
91
|
if api_request?
|
|
83
92
|
json_response('edit')
|
|
84
|
-
elsif
|
|
93
|
+
elsif Bulkrax.hyrax_loaded?
|
|
85
94
|
add_importer_breadcrumbs
|
|
86
95
|
add_breadcrumb @importer.name, bulkrax.importer_path(@importer.id)
|
|
87
96
|
add_breadcrumb t(:'bulkrax.headings.edit_importer')
|
|
@@ -92,28 +101,24 @@ module Bulkrax
|
|
|
92
101
|
# rubocop:disable Metrics/MethodLength
|
|
93
102
|
# rubocop:disable Metrics/AbcSize
|
|
94
103
|
def create
|
|
104
|
+
# rubocop:disable Style/IfInsideElse
|
|
95
105
|
if api_request?
|
|
96
106
|
return return_json_response unless valid_create_params?
|
|
107
|
+
# authenticate_and_authorize! doesn't build for API requests; build it here
|
|
108
|
+
@importer ||= Importer.new(importer_params)
|
|
97
109
|
end
|
|
110
|
+
@importer.user_id = current_user.id
|
|
98
111
|
uploads = uploaded_files_scope
|
|
99
112
|
file = file_param
|
|
100
113
|
cloud_files = cloud_params
|
|
101
114
|
|
|
102
|
-
@importer = Importer.new(importer_params)
|
|
103
115
|
field_mapping_params
|
|
104
116
|
@importer.validate_only = true if params[:commit] == 'Create and Validate'
|
|
105
117
|
# the following line is needed to handle updating remote files of a FileSet
|
|
106
118
|
# on a new import otherwise it only gets updated during the update path
|
|
107
119
|
@importer.parser_fields['update_files'] = true if params[:commit] == 'Create and Import'
|
|
108
|
-
|
|
109
|
-
# Reject an unsafe cloud selection before the importer is ever
|
|
110
|
-
# persisted, rather than after save when a row would already exist.
|
|
111
|
-
preflight_cloud_files!(cloud_files)
|
|
112
|
-
|
|
113
120
|
if @importer.save
|
|
114
|
-
|
|
115
|
-
dispatch_cloud_downloads!
|
|
116
|
-
|
|
121
|
+
files_for_import(file, cloud_files, uploads)
|
|
117
122
|
if params[:commit] == 'Create and Import'
|
|
118
123
|
Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id)
|
|
119
124
|
render_request('Importer was successfully created and import has been queued.')
|
|
@@ -124,10 +129,13 @@ module Bulkrax
|
|
|
124
129
|
render_request('Importer was successfully created.')
|
|
125
130
|
end
|
|
126
131
|
else
|
|
127
|
-
|
|
132
|
+
if api_request?
|
|
133
|
+
json_response('create', :unprocessable_entity)
|
|
134
|
+
else
|
|
135
|
+
render :new
|
|
136
|
+
end
|
|
128
137
|
end
|
|
129
|
-
|
|
130
|
-
reject_unsafe_cloud_files(e)
|
|
138
|
+
# rubocop:enable Style/IfInsideElse
|
|
131
139
|
end
|
|
132
140
|
# rubocop:enable Metrics/AbcSize
|
|
133
141
|
|
|
@@ -145,19 +153,11 @@ module Bulkrax
|
|
|
145
153
|
# Skipped during a continue
|
|
146
154
|
field_mapping_params if params[:importer][:parser_fields].present?
|
|
147
155
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
# below then leaves the importer's persisted attributes untouched.
|
|
151
|
-
@importer.assign_attributes(importer_params)
|
|
152
|
-
preflight_cloud_files!(cloud_files)
|
|
153
|
-
|
|
154
|
-
if @importer.save
|
|
155
|
-
return render_update_failure if files_for_import(file, cloud_files, uploads) && !@importer.save
|
|
156
|
-
return render_update_failure if params[:commit] != 'Update Importer' && !set_files_parser_fields
|
|
157
|
-
dispatch_cloud_downloads!
|
|
158
|
-
|
|
156
|
+
if @importer.update(importer_params)
|
|
157
|
+
files_for_import(file, cloud_files, uploads)
|
|
159
158
|
# do not perform the import
|
|
160
159
|
unless params[:commit] == 'Update Importer'
|
|
160
|
+
set_files_parser_fields
|
|
161
161
|
Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id, update_harvest)
|
|
162
162
|
end
|
|
163
163
|
if api_request?
|
|
@@ -166,10 +166,12 @@ module Bulkrax
|
|
|
166
166
|
redirect_to importers_path, notice: 'Importer was successfully updated.'
|
|
167
167
|
end
|
|
168
168
|
else
|
|
169
|
-
|
|
169
|
+
if api_request?
|
|
170
|
+
json_response('update', :unprocessable_entity, 'Something went wrong.')
|
|
171
|
+
else
|
|
172
|
+
render :edit
|
|
173
|
+
end
|
|
170
174
|
end
|
|
171
|
-
rescue Bulkrax::UnsafeCloudSourceError, Bulkrax::UnsafePathError => e
|
|
172
|
-
reject_unsafe_cloud_files(e)
|
|
173
175
|
end
|
|
174
176
|
# rubocop:enable all
|
|
175
177
|
# rubocop:enable Metrics/MethodLength
|
|
@@ -186,7 +188,6 @@ module Bulkrax
|
|
|
186
188
|
|
|
187
189
|
# PUT /importers/1
|
|
188
190
|
def continue
|
|
189
|
-
@importer = Importer.find(params[:importer_id])
|
|
190
191
|
params[:importer] = { name: @importer.name }
|
|
191
192
|
@importer.validate_only = false
|
|
192
193
|
update
|
|
@@ -194,8 +195,7 @@ module Bulkrax
|
|
|
194
195
|
|
|
195
196
|
# GET /importer/1/upload_corrected_entries
|
|
196
197
|
def upload_corrected_entries
|
|
197
|
-
|
|
198
|
-
return unless defined?(::Hyrax)
|
|
198
|
+
return unless Bulkrax.hyrax_loaded?
|
|
199
199
|
add_breadcrumb t(:'hyrax.controls.home'), main_app.root_path
|
|
200
200
|
add_breadcrumb t(:'hyrax.dashboard.breadcrumbs.admin'), hyrax.dashboard_path
|
|
201
201
|
add_breadcrumb t(:'bulkrax.headings.importers'), bulkrax.importers_path
|
|
@@ -206,7 +206,6 @@ module Bulkrax
|
|
|
206
206
|
# POST /importer/1/upload_corrected_entries_file
|
|
207
207
|
def upload_corrected_entries_file
|
|
208
208
|
file = params[:importer][:parser_fields].delete(:file)
|
|
209
|
-
@importer = Importer.find(params[:importer_id])
|
|
210
209
|
if file.present?
|
|
211
210
|
@importer[:parser_fields]['partial_import_file_path'] = @importer.parser.write_partial_import_file(file)
|
|
212
211
|
@importer.save
|
|
@@ -251,18 +250,12 @@ module Bulkrax
|
|
|
251
250
|
|
|
252
251
|
# GET /importers/1/export_errors
|
|
253
252
|
def export_errors
|
|
254
|
-
@importer = Importer.find(params[:importer_id])
|
|
255
253
|
@importer.write_errored_entries_file
|
|
256
254
|
send_content
|
|
257
255
|
end
|
|
258
256
|
|
|
259
257
|
private
|
|
260
258
|
|
|
261
|
-
# Use callbacks to share common setup or constraints between actions.
|
|
262
|
-
def set_importer
|
|
263
|
-
@importer = Importer.find(params[:id] || params[:importer_id])
|
|
264
|
-
end
|
|
265
|
-
|
|
266
259
|
def importable_params
|
|
267
260
|
params.except(:selected_files)
|
|
268
261
|
end
|
|
@@ -276,7 +269,6 @@ module Bulkrax
|
|
|
276
269
|
importable_params.require(:importer).permit(
|
|
277
270
|
:name,
|
|
278
271
|
:admin_set_id,
|
|
279
|
-
:user_id,
|
|
280
272
|
:frequency,
|
|
281
273
|
:parser_klass,
|
|
282
274
|
:limit,
|
|
@@ -341,36 +333,6 @@ module Bulkrax
|
|
|
341
333
|
'text/csv'
|
|
342
334
|
end
|
|
343
335
|
|
|
344
|
-
# Reports an unsafe cloud-file batch without leaking the error's class
|
|
345
|
-
# or backtrace (which could include a provider auth header) to the
|
|
346
|
-
# response; `e.message` is always one of our own static, credential-free
|
|
347
|
-
# strings (see Bulkrax::CloudSourceValidator / Bulkrax::SafePath).
|
|
348
|
-
def reject_unsafe_cloud_files(e)
|
|
349
|
-
Rails.logger.error("Bulkrax: rejected unsafe cloud file batch for importer #{@importer&.id}: #{e.class}: #{e.message}")
|
|
350
|
-
if api_request?
|
|
351
|
-
json_response('create', :unprocessable_entity, e.message)
|
|
352
|
-
else
|
|
353
|
-
flash[:error] = e.message
|
|
354
|
-
redirect_back fallback_location: bulkrax.importers_path
|
|
355
|
-
end
|
|
356
|
-
end
|
|
357
|
-
|
|
358
|
-
def render_create_failure
|
|
359
|
-
if api_request?
|
|
360
|
-
json_response('create', :unprocessable_entity)
|
|
361
|
-
else
|
|
362
|
-
render :new
|
|
363
|
-
end
|
|
364
|
-
end
|
|
365
|
-
|
|
366
|
-
def render_update_failure
|
|
367
|
-
if api_request?
|
|
368
|
-
json_response('update', :unprocessable_entity, 'Something went wrong.')
|
|
369
|
-
else
|
|
370
|
-
render :edit
|
|
371
|
-
end
|
|
372
|
-
end
|
|
373
|
-
|
|
374
336
|
def render_request(message, validate_only = false)
|
|
375
337
|
if api_request?
|
|
376
338
|
json_response('create', :created, message)
|
|
@@ -386,8 +348,6 @@ module Bulkrax
|
|
|
386
348
|
params[:commit] == 'Update and Harvest Updated Items'
|
|
387
349
|
end
|
|
388
350
|
|
|
389
|
-
# @return [Boolean] the result of the save, so the caller can treat a
|
|
390
|
-
# failure here like an ordinary validation failure
|
|
391
351
|
def set_files_parser_fields
|
|
392
352
|
@importer.parser_fields['update_files'] =
|
|
393
353
|
@importer.parser_fields['replace_files'] =
|
|
@@ -400,16 +360,12 @@ module Bulkrax
|
|
|
400
360
|
elsif params[:commit] == 'Remove and Rerun'
|
|
401
361
|
@importer.parser_fields['remove_and_rerun'] = true
|
|
402
362
|
elsif params[:commit] == 'Update and Harvest Updated Items'
|
|
403
|
-
return
|
|
363
|
+
return
|
|
404
364
|
else
|
|
405
365
|
@importer.parser_fields['metadata_only'] = true
|
|
406
366
|
end
|
|
407
367
|
@importer.save
|
|
408
368
|
end
|
|
409
|
-
|
|
410
|
-
def check_permissions
|
|
411
|
-
raise CanCan::AccessDenied unless current_ability.can_import_works?
|
|
412
|
-
end
|
|
413
369
|
end
|
|
414
370
|
# rubocop:enable Metrics/ClassLength
|
|
415
371
|
end
|
|
@@ -83,7 +83,7 @@ module Bulkrax
|
|
|
83
83
|
@exporter_table_search
|
|
84
84
|
end
|
|
85
85
|
|
|
86
|
-
def format_importers(importers)
|
|
86
|
+
def format_importers(importers, filtered_count = Bulkrax::Importer.accessible_by(current_ability).count)
|
|
87
87
|
result = importers.map do |i|
|
|
88
88
|
{
|
|
89
89
|
name: view_context.link_to(i.name, view_context.importer_path(i)),
|
|
@@ -102,12 +102,12 @@ module Bulkrax
|
|
|
102
102
|
end
|
|
103
103
|
{
|
|
104
104
|
data: result,
|
|
105
|
-
recordsTotal: Bulkrax::Importer.count,
|
|
106
|
-
recordsFiltered:
|
|
105
|
+
recordsTotal: api_request? ? Bulkrax::Importer.count : Bulkrax::Importer.accessible_by(current_ability).count,
|
|
106
|
+
recordsFiltered: filtered_count
|
|
107
107
|
}
|
|
108
108
|
end
|
|
109
109
|
|
|
110
|
-
def format_exporters(exporters)
|
|
110
|
+
def format_exporters(exporters, filtered_count = Bulkrax::Exporter.accessible_by(current_ability).count)
|
|
111
111
|
result = exporters.map do |e|
|
|
112
112
|
{
|
|
113
113
|
name: view_context.link_to(e.name, view_context.exporter_path(e)),
|
|
@@ -119,8 +119,8 @@ module Bulkrax
|
|
|
119
119
|
end
|
|
120
120
|
{
|
|
121
121
|
data: result,
|
|
122
|
-
recordsTotal: Bulkrax::Exporter.count,
|
|
123
|
-
recordsFiltered:
|
|
122
|
+
recordsTotal: Bulkrax::Exporter.accessible_by(current_ability).count,
|
|
123
|
+
recordsFiltered: filtered_count
|
|
124
124
|
}
|
|
125
125
|
end
|
|
126
126
|
|
|
@@ -146,8 +146,10 @@ module Bulkrax
|
|
|
146
146
|
def entry_util_links(e, item)
|
|
147
147
|
links = []
|
|
148
148
|
links << view_context.link_to(view_context.raw('<span class="fa fa-info-circle"></span>'), view_context.item_entry_path(item, e))
|
|
149
|
-
|
|
150
|
-
|
|
149
|
+
if can?(:update, item)
|
|
150
|
+
links << "<a class='fa fa-repeat' data-toggle='modal' data-target='#bulkraxItemModal' data-entry-id='#{e.id}'></a>" if view_context.an_importer?(item)
|
|
151
|
+
links << view_context.link_to(view_context.raw('<span class="fa fa-trash"></span>'), view_context.item_entry_path(item, e), method: :delete, data: { confirm: 'This will delete the entry and any work associated with it. Are you sure?' })
|
|
152
|
+
end
|
|
151
153
|
links.join(" ")
|
|
152
154
|
end
|
|
153
155
|
|
|
@@ -166,16 +168,20 @@ module Bulkrax
|
|
|
166
168
|
def importer_util_links(i)
|
|
167
169
|
links = []
|
|
168
170
|
links << view_context.link_to(view_context.raw('<span class="fa fa-info-circle"></span>'), importer_path(i))
|
|
169
|
-
|
|
170
|
-
|
|
171
|
+
if can?(:update, i)
|
|
172
|
+
links << view_context.link_to(view_context.raw('<span class="fa fa-pencil"></span>'), edit_importer_path(i))
|
|
173
|
+
links << view_context.link_to(view_context.raw('<span class="fa fa-remove"></span>'), i, method: :delete, data: { confirm: 'Are you sure?' })
|
|
174
|
+
end
|
|
171
175
|
links.join(" ")
|
|
172
176
|
end
|
|
173
177
|
|
|
174
178
|
def exporter_util_links(i)
|
|
175
179
|
links = []
|
|
176
180
|
links << view_context.link_to(view_context.raw('<span class="fa fa-info-circle"></span>'), exporter_path(i))
|
|
177
|
-
|
|
178
|
-
|
|
181
|
+
if can?(:update, i)
|
|
182
|
+
links << view_context.link_to(view_context.raw('<span class="fa fa-pencil"></span>'), edit_exporter_path(i), data: { turbolinks: false })
|
|
183
|
+
links << view_context.link_to(view_context.raw('<span class="fa fa-remove"></span>'), i, method: :delete, data: { confirm: 'Are you sure?' })
|
|
184
|
+
end
|
|
179
185
|
links.join(" ")
|
|
180
186
|
end
|
|
181
187
|
|
|
@@ -43,7 +43,7 @@ module Bulkrax
|
|
|
43
43
|
end
|
|
44
44
|
|
|
45
45
|
def uploaded_files_scope
|
|
46
|
-
return [] unless
|
|
46
|
+
return [] unless Bulkrax.hyrax_loaded?
|
|
47
47
|
|
|
48
48
|
base = Hyrax::UploadedFile.where(id: params[:uploaded_files])
|
|
49
49
|
if respond_to?(:current_user) && current_user.present?
|
|
@@ -139,38 +139,21 @@ module Bulkrax
|
|
|
139
139
|
csv_by_depth[shallowest_depth]
|
|
140
140
|
end
|
|
141
141
|
|
|
142
|
-
#
|
|
143
|
-
# safety — before the importer is created or updated at all. Must not
|
|
144
|
-
# touch the importer's workspace (no path_for_import, no directory
|
|
145
|
-
# creation, no download): for #create that workspace doesn't exist
|
|
146
|
-
# yet, and for #update it must not be assumed safe to touch before the
|
|
147
|
-
# proposed attributes are known to be acceptable.
|
|
148
|
-
#
|
|
149
|
-
# @param cloud_files [Hash, nil] cloud file specs from browse-everything
|
|
150
|
-
# @raise [Bulkrax::UnsafePathError, Bulkrax::UnsafeCloudSourceError] on the first unsafe entry
|
|
151
|
-
def preflight_cloud_files!(cloud_files)
|
|
152
|
-
return if cloud_files.blank?
|
|
153
|
-
|
|
154
|
-
cloud_files.values.each { |file| Bulkrax::CloudSourceValidator.preflight!(file) }
|
|
155
|
-
end
|
|
156
|
-
|
|
157
|
-
# Assigns files/cloud selections onto the importer record's in-memory
|
|
158
|
-
# parser fields, but does not save: the caller (ImportersController)
|
|
159
|
-
# owns the second save, so it can check its result and avoid rendering
|
|
160
|
-
# success or enqueueing an import on failure.
|
|
161
|
-
#
|
|
142
|
+
# Persists uploaded file(s) and/or cloud files onto the importer record.
|
|
162
143
|
# @param file [ActionDispatch::Http::UploadedFile, nil] a directly uploaded file
|
|
163
144
|
# @param cloud_files [Hash, nil] cloud file paths from browse-everything
|
|
164
145
|
# @param uploads [ActiveRecord::Relation, Array, nil] Hyrax::UploadedFile records
|
|
165
|
-
# @return [Boolean] true if any parser field was touched and a second save is needed
|
|
166
146
|
def files_for_import(file, cloud_files, uploads)
|
|
167
|
-
|
|
168
|
-
return false if file.blank? && cloud_files.blank? && uploads.blank?
|
|
147
|
+
return if file.blank? && cloud_files.blank? && uploads.blank?
|
|
169
148
|
|
|
170
149
|
@importer[:parser_fields]['import_file_path'] = @importer.parser.write_import_file(file) if file.present?
|
|
171
150
|
if cloud_files.present?
|
|
172
151
|
@importer[:parser_fields]['cloud_file_paths'] = cloud_files
|
|
173
|
-
|
|
152
|
+
# For BagIt, there will only be one bag, so we get the file_path back and set import_file_path
|
|
153
|
+
# For CSV, we expect only file uploads, so we won't get the file_path back
|
|
154
|
+
# and we expect the import_file_path to be set already
|
|
155
|
+
target = @importer.parser.retrieve_cloud_files(cloud_files, @importer)
|
|
156
|
+
@importer[:parser_fields]['import_file_path'] = target if target.present?
|
|
174
157
|
end
|
|
175
158
|
|
|
176
159
|
if uploads.present?
|
|
@@ -179,36 +162,7 @@ module Bulkrax
|
|
|
179
162
|
end
|
|
180
163
|
end
|
|
181
164
|
|
|
182
|
-
|
|
183
|
-
end
|
|
184
|
-
|
|
185
|
-
# For a parser that separates planning from dispatch (currently only
|
|
186
|
-
# CsvParser), resolves and validates the whole batch and records the
|
|
187
|
-
# target paths, deferring enqueue until {#dispatch_cloud_downloads!} is
|
|
188
|
-
# called after the controller persists those paths. Other parsers
|
|
189
|
-
# (e.g. BagitParser) already complete cloud retrieval synchronously
|
|
190
|
-
# and return the resolved path directly, so there's nothing left to
|
|
191
|
-
# dispatch.
|
|
192
|
-
def assign_cloud_file_paths(cloud_files)
|
|
193
|
-
parser = @importer.parser
|
|
194
|
-
if parser.respond_to?(:plan_cloud_files)
|
|
195
|
-
@cloud_download_plan = parser.plan_cloud_files(cloud_files)
|
|
196
|
-
@importer[:parser_fields]['original_file_paths'] =
|
|
197
|
-
@cloud_download_plan.map { |_file, _relative_filename, target_file| target_file }
|
|
198
|
-
else
|
|
199
|
-
target = parser.retrieve_cloud_files(cloud_files, @importer)
|
|
200
|
-
@importer[:parser_fields]['import_file_path'] = target if target.present?
|
|
201
|
-
end
|
|
202
|
-
end
|
|
203
|
-
|
|
204
|
-
# Dispatches a plan prepared by {#assign_cloud_file_paths}, if any.
|
|
205
|
-
# Called by the controller only after it has successfully persisted
|
|
206
|
-
# the resolved paths, so a download is never enqueued for a path the
|
|
207
|
-
# importer doesn't actually have on record.
|
|
208
|
-
def dispatch_cloud_downloads!
|
|
209
|
-
return if @cloud_download_plan.blank?
|
|
210
|
-
|
|
211
|
-
@importer.parser.dispatch_cloud_files!(@cloud_download_plan, @importer)
|
|
165
|
+
@importer.save
|
|
212
166
|
end
|
|
213
167
|
|
|
214
168
|
def write_files(files)
|
|
@@ -168,8 +168,8 @@ module Bulkrax
|
|
|
168
168
|
end
|
|
169
169
|
|
|
170
170
|
def self.solr_name(field_name)
|
|
171
|
-
if defined?(Hyrax)
|
|
172
|
-
Hyrax.index_field_mapper.solr_name(field_name)
|
|
171
|
+
if defined?(Hyrax) && Hyrax.respond_to?(:config) && Hyrax.config.respond_to?(:index_field_mapper)
|
|
172
|
+
Hyrax.config.index_field_mapper.solr_name(field_name)
|
|
173
173
|
else
|
|
174
174
|
ActiveFedora.index_field_mapper.solr_name(field_name)
|
|
175
175
|
end
|
|
@@ -280,11 +280,14 @@ module Bulkrax
|
|
|
280
280
|
# @param admin_set_id [String, nil] admin set used to resolve contexts
|
|
281
281
|
# @return [Array<String>]
|
|
282
282
|
def self.schema_properties(klass:, admin_set_id: nil)
|
|
283
|
-
|
|
283
|
+
entry = schema_cache_entry(klass: klass, admin_set_id: admin_set_id)
|
|
284
|
+
entry[:properties] ||= entry[:schema].map { |k| k.name.to_s }.freeze
|
|
284
285
|
end
|
|
285
286
|
|
|
286
287
|
##
|
|
287
|
-
# Returns the schema for a model, memoized per (klass, admin_set_id)
|
|
288
|
+
# Returns the schema for a model, memoized per (tenant, klass, admin_set_id)
|
|
289
|
+
# for the current request or job only, so a new metadata profile or a
|
|
290
|
+
# change to an admin set's contexts is picked up without a restart.
|
|
288
291
|
# Delegates to +Hyrax.schema_for+ when available so that context-gated
|
|
289
292
|
# properties are included without Bulkrax knowing about flexibility internals.
|
|
290
293
|
#
|
|
@@ -292,16 +295,25 @@ module Bulkrax
|
|
|
292
295
|
# @param admin_set_id [String, nil]
|
|
293
296
|
# @return [Dry::Types::Hash]
|
|
294
297
|
def self.cached_schema_for(klass:, admin_set_id: nil)
|
|
295
|
-
|
|
296
|
-
key = [klass.name, admin_set_id].compact.join('|')
|
|
297
|
-
@cached_schema_map[key] ||=
|
|
298
|
-
if admin_set_id.present? && defined?(Hyrax) && Hyrax.respond_to?(:schema_for)
|
|
299
|
-
Hyrax.schema_for(klass: klass, admin_set_id: admin_set_id)
|
|
300
|
-
else
|
|
301
|
-
klass.new.singleton_class.schema || klass.schema
|
|
302
|
-
end
|
|
298
|
+
schema_cache_entry(klass: klass, admin_set_id: admin_set_id)[:schema]
|
|
303
299
|
end
|
|
304
300
|
|
|
301
|
+
def self.schema_cache_entry(klass:, admin_set_id:)
|
|
302
|
+
tenant = defined?(Apartment::Tenant) ? Apartment::Tenant.current : 'single'
|
|
303
|
+
Bulkrax::Current.schema_cache ||= {}
|
|
304
|
+
Bulkrax::Current.schema_cache[[tenant, klass, admin_set_id]] ||= { schema: resolve_schema(klass: klass, admin_set_id: admin_set_id) }
|
|
305
|
+
end
|
|
306
|
+
private_class_method :schema_cache_entry
|
|
307
|
+
|
|
308
|
+
def self.resolve_schema(klass:, admin_set_id:)
|
|
309
|
+
if admin_set_id.present? && defined?(Hyrax) && Hyrax.respond_to?(:schema_for)
|
|
310
|
+
Hyrax.schema_for(klass: klass, admin_set_id: admin_set_id)
|
|
311
|
+
else
|
|
312
|
+
klass.new.singleton_class.schema || klass.schema
|
|
313
|
+
end
|
|
314
|
+
end
|
|
315
|
+
private_class_method :resolve_schema
|
|
316
|
+
|
|
305
317
|
def self.ordered_file_sets_for(object)
|
|
306
318
|
return [] if object.blank?
|
|
307
319
|
|
|
@@ -434,7 +446,7 @@ module Bulkrax
|
|
|
434
446
|
|
|
435
447
|
file_attrs = []
|
|
436
448
|
uploads.each_with_index do |f, index|
|
|
437
|
-
file_attrs <<
|
|
449
|
+
file_attrs << { uploaded_file_id: f["id"].to_s, filename: files[index]["file_name"] }.merge(additional_attributes[index])
|
|
438
450
|
end
|
|
439
451
|
file_attrs.compact.uniq
|
|
440
452
|
end
|
|
@@ -501,7 +513,7 @@ module Bulkrax
|
|
|
501
513
|
@permitted_attributes ||= begin
|
|
502
514
|
bare = base_permitted_attributes + if klass.respond_to?(:schema)
|
|
503
515
|
admin_set_id = attributes[:admin_set_id] || attributes['admin_set_id']
|
|
504
|
-
Bulkrax::ValkyrieObjectFactory.schema_properties(klass: klass, admin_set_id: admin_set_id)
|
|
516
|
+
Bulkrax::ValkyrieObjectFactory.schema_properties(klass: klass, admin_set_id: admin_set_id).map(&:to_sym)
|
|
505
517
|
else
|
|
506
518
|
klass.properties.keys.map(&:to_sym)
|
|
507
519
|
end
|
|
@@ -590,18 +602,18 @@ module Bulkrax
|
|
|
590
602
|
|
|
591
603
|
# @Override Destroy existing files with Hyrax::Transactions
|
|
592
604
|
def destroy_existing_files(object: @object)
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
existing_files.each do |fs|
|
|
605
|
+
destroyed_ids = []
|
|
606
|
+
Hyrax.custom_queries.find_child_file_sets(resource: object).each do |fs|
|
|
597
607
|
transactions["file_set.destroy"]
|
|
598
608
|
.with_step_args("file_set.remove_from_work" => { user: @user },
|
|
599
609
|
"file_set.delete" => { user: @user })
|
|
600
610
|
.call(fs)
|
|
601
611
|
.value!
|
|
612
|
+
destroyed_ids << fs.id
|
|
602
613
|
end
|
|
614
|
+
return if destroyed_ids.empty?
|
|
603
615
|
|
|
604
|
-
object.member_ids
|
|
616
|
+
object.member_ids -= destroyed_ids
|
|
605
617
|
object.rendering_ids = []
|
|
606
618
|
object.representative_id = nil
|
|
607
619
|
object.thumbnail_id = nil
|
|
@@ -114,7 +114,7 @@ module Bulkrax
|
|
|
114
114
|
# We can use Hyrax's lock manager when we have one available.
|
|
115
115
|
# However it's not certain that this is actually working, so to be
|
|
116
116
|
# as safe as possible, we will reload resources before we update.
|
|
117
|
-
if
|
|
117
|
+
if Bulkrax.hyrax_loaded?
|
|
118
118
|
include Hyrax::Lockable
|
|
119
119
|
|
|
120
120
|
def conditionally_acquire_lock_for(*args, &block)
|