bulkrax 9.5.1 → 10.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +49 -0
  3. data/app/controllers/bulkrax/application_controller.rb +83 -0
  4. data/app/controllers/bulkrax/entries_controller.rb +16 -13
  5. data/app/controllers/bulkrax/exporters_controller.rb +21 -25
  6. data/app/controllers/bulkrax/guided_imports_controller.rb +4 -8
  7. data/app/controllers/bulkrax/importers_controller.rb +84 -46
  8. data/app/controllers/concerns/bulkrax/datatables_behavior.rb +18 -12
  9. data/app/controllers/concerns/bulkrax/importer_file_handler.rb +55 -9
  10. data/app/errors/bulkrax/unsafe_cloud_source_error.rb +11 -0
  11. data/app/errors/bulkrax/unsafe_path_error.rb +8 -0
  12. data/app/factories/bulkrax/object_factory.rb +2 -2
  13. data/app/factories/bulkrax/valkyrie_object_factory.rb +29 -17
  14. data/app/helpers/bulkrax/validation_helper.rb +1 -1
  15. data/app/jobs/bulkrax/create_relationships_job.rb +1 -1
  16. data/app/jobs/bulkrax/download_cloud_file_job.rb +156 -21
  17. data/app/jobs/bulkrax/import_collection_job.rb +9 -2
  18. data/app/matchers/bulkrax/application_matcher.rb +5 -2
  19. data/app/models/bulkrax/csv_entry.rb +1 -1
  20. data/app/models/bulkrax/current.rb +7 -0
  21. data/app/models/bulkrax/export_scope.rb +21 -0
  22. data/app/models/bulkrax/exporter.rb +2 -2
  23. data/app/models/bulkrax/importer.rb +1 -1
  24. data/app/models/concerns/bulkrax/ability_behavior.rb +139 -0
  25. data/app/models/concerns/bulkrax/has_matchers.rb +1 -1
  26. data/app/models/concerns/bulkrax/import_behavior.rb +3 -3
  27. data/app/parsers/bulkrax/application_parser.rb +11 -11
  28. data/app/parsers/bulkrax/application_parser.rb.orig +545 -0
  29. data/app/parsers/bulkrax/bagit_parser.rb +10 -3
  30. data/app/parsers/bulkrax/csv_parser.rb +59 -27
  31. data/app/parsers/bulkrax/oai_dc_parser.rb +1 -1
  32. data/app/parsers/bulkrax/parser_export_record_set.rb +26 -8
  33. data/app/parsers/concerns/bulkrax/csv_parser/csv_template_generation.rb +1 -1
  34. data/app/parsers/concerns/bulkrax/csv_parser/csv_validation.rb +1 -2
  35. data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_helpers.rb +18 -1
  36. data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_hierarchy.rb +2 -2
  37. data/app/search_builders/bulkrax/export_search_builder.rb +9 -0
  38. data/app/services/bulkrax/cloud_source_validator.rb +123 -0
  39. data/app/services/bulkrax/csv_template/column_builder.rb +2 -2
  40. data/app/services/bulkrax/csv_template/mapping_manager.rb +9 -7
  41. data/app/services/bulkrax/csv_template/schema_analyzer.rb +1 -5
  42. data/app/services/bulkrax/safe_path.rb +129 -0
  43. data/config/locales/bulkrax.en.yml +2 -0
  44. data/lib/bulkrax/engine.rb +1 -1
  45. data/lib/bulkrax/version.rb +1 -1
  46. data/lib/bulkrax.rb +77 -10
  47. data/lib/generators/bulkrax/install_generator.rb +22 -10
  48. data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb +1 -1
  49. metadata +58 -14
  50. data/app/factories/bulkrax/valkyrize-hyku.code-workspace +0 -19
  51. data/lib/wings/custom_queries/find_by_source_identifier.rb +0 -41
@@ -3,7 +3,7 @@
3
3
  module Bulkrax
4
4
  # rubocop:disable Metrics/ClassLength
5
5
  class ImportersController < ::Bulkrax::ApplicationController
6
- include Hyrax::ThemedLayoutController if defined?(::Hyrax)
6
+ include Hyrax::ThemedLayoutController if Bulkrax.hyrax_loaded?
7
7
  include Bulkrax::DownloadBehavior
8
8
  include Bulkrax::API
9
9
  include Bulkrax::DatatablesBehavior
@@ -11,29 +11,39 @@ module Bulkrax
11
11
  include Bulkrax::ImporterFileHandler
12
12
 
13
13
  protect_from_forgery unless: -> { api_request? }
14
- before_action :token_authenticate!, if: -> { api_request? }, only: [:create, :update, :delete]
15
- before_action :authenticate_user!, unless: -> { api_request? }
16
- before_action :check_permissions
17
- before_action :set_importer, only: [:show, :entry_table, :edit, :update, :destroy, :original_file]
18
- with_themed_layout 'dashboard' if defined?(::Hyrax)
14
+ # Authentication, record loading and authorization for every action.
15
+ before_action { authenticate_and_authorize!(Bulkrax::Importer, :can_import_works?) }
16
+ with_themed_layout 'dashboard' if Bulkrax.hyrax_loaded?
19
17
 
20
18
  # GET /importers
21
19
  def index
22
20
  # NOTE: We're paginating this in the browser.
23
21
  if api_request?
22
+ # TODO(#1237): Scope API index by token owner once token-to-user wiring
23
+ # is in place. Until then, API clients see all importers
24
+ # to preserve backward-compatibility with existing API consumers.
24
25
  @importers = Importer.order(created_at: :desc).all
25
26
  json_response('index')
26
- elsif defined?(::Hyrax)
27
+ elsif Bulkrax.hyrax_loaded?
27
28
  add_importer_breadcrumbs
28
29
  end
29
30
  end
30
31
 
31
32
  def importer_table
32
33
  order = table_order.presence || Arel.sql('last_imported_at DESC NULLS LAST')
33
- @importers = Importer.order(order).page(table_page).per(table_per_page)
34
+ # TODO(#1237): API requests bypass ownership scoping here too.
35
+ @importers = if api_request?
36
+ Importer.all
37
+ else
38
+ Importer.accessible_by(current_ability)
39
+ end
34
40
  @importers = @importers.where(importer_table_search) if importer_table_search.present?
41
+ # Count the filtered relation before applying pagination so the UI receives
42
+ # the total number of matching importers, not just the total number of importers.
43
+ filtered_count = @importers.count
44
+ @importers = @importers.order(order).page(table_page).per(table_per_page)
35
45
  respond_to do |format|
36
- format.json { render json: format_importers(@importers) }
46
+ format.json { render json: format_importers(@importers, filtered_count) }
37
47
  end
38
48
  end
39
49
 
@@ -41,7 +51,7 @@ module Bulkrax
41
51
  def show
42
52
  if api_request?
43
53
  json_response('show')
44
- elsif defined?(::Hyrax)
54
+ elsif Bulkrax.hyrax_loaded?
45
55
  add_importer_breadcrumbs
46
56
  add_breadcrumb @importer.name
47
57
  end
@@ -58,10 +68,9 @@ module Bulkrax
58
68
 
59
69
  # GET /importers/new
60
70
  def new
61
- @importer = Importer.new
62
71
  if api_request?
63
72
  json_response('new')
64
- elsif defined?(::Hyrax)
73
+ elsif Bulkrax.hyrax_loaded?
65
74
  add_importer_breadcrumbs
66
75
  add_breadcrumb t(:'bulkrax.headings.new_importer')
67
76
  end
@@ -81,7 +90,7 @@ module Bulkrax
81
90
  def edit
82
91
  if api_request?
83
92
  json_response('edit')
84
- elsif defined?(::Hyrax)
93
+ elsif Bulkrax.hyrax_loaded?
85
94
  add_importer_breadcrumbs
86
95
  add_breadcrumb @importer.name, bulkrax.importer_path(@importer.id)
87
96
  add_breadcrumb t(:'bulkrax.headings.edit_importer')
@@ -92,22 +101,30 @@ module Bulkrax
92
101
  # rubocop:disable Metrics/MethodLength
93
102
  # rubocop:disable Metrics/AbcSize
94
103
  def create
95
- # rubocop:disable Style/IfInsideElse
96
104
  if api_request?
97
105
  return return_json_response unless valid_create_params?
106
+ # authenticate_and_authorize! doesn't build for API requests; build it here
107
+ @importer ||= Importer.new(importer_params)
98
108
  end
109
+ @importer.user_id = current_user.id
99
110
  uploads = uploaded_files_scope
100
111
  file = file_param
101
112
  cloud_files = cloud_params
102
113
 
103
- @importer = Importer.new(importer_params)
104
114
  field_mapping_params
105
115
  @importer.validate_only = true if params[:commit] == 'Create and Validate'
106
116
  # the following line is needed to handle updating remote files of a FileSet
107
117
  # on a new import otherwise it only gets updated during the update path
108
118
  @importer.parser_fields['update_files'] = true if params[:commit] == 'Create and Import'
119
+
120
+ # Reject an unsafe cloud selection before the importer is ever
121
+ # persisted, rather than after save when a row would already exist.
122
+ preflight_cloud_files!(cloud_files)
123
+
109
124
  if @importer.save
110
- files_for_import(file, cloud_files, uploads)
125
+ return render_create_failure if files_for_import(file, cloud_files, uploads) && !@importer.save
126
+ dispatch_cloud_downloads!
127
+
111
128
  if params[:commit] == 'Create and Import'
112
129
  Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id)
113
130
  render_request('Importer was successfully created and import has been queued.')
@@ -118,13 +135,10 @@ module Bulkrax
118
135
  render_request('Importer was successfully created.')
119
136
  end
120
137
  else
121
- if api_request?
122
- json_response('create', :unprocessable_entity)
123
- else
124
- render :new
125
- end
138
+ render_create_failure
126
139
  end
127
- # rubocop:enable Style/IfInsideElse
140
+ rescue Bulkrax::UnsafeCloudSourceError, Bulkrax::UnsafePathError => e
141
+ reject_unsafe_cloud_files(e)
128
142
  end
129
143
  # rubocop:enable Metrics/AbcSize
130
144
 
@@ -142,11 +156,19 @@ module Bulkrax
142
156
  # Skipped during a continue
143
157
  field_mapping_params if params[:importer][:parser_fields].present?
144
158
 
145
- if @importer.update(importer_params)
146
- files_for_import(file, cloud_files, uploads)
159
+ # Assign the proposed attributes in memory first, so an unsafe cloud
160
+ # selection is rejected before anything is persisted; the rescue
161
+ # below then leaves the importer's persisted attributes untouched.
162
+ @importer.assign_attributes(importer_params)
163
+ preflight_cloud_files!(cloud_files)
164
+
165
+ if @importer.save
166
+ return render_update_failure if files_for_import(file, cloud_files, uploads) && !@importer.save
167
+ return render_update_failure if params[:commit] != 'Update Importer' && !set_files_parser_fields
168
+ dispatch_cloud_downloads!
169
+
147
170
  # do not perform the import
148
171
  unless params[:commit] == 'Update Importer'
149
- set_files_parser_fields
150
172
  Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id, update_harvest)
151
173
  end
152
174
  if api_request?
@@ -155,12 +177,10 @@ module Bulkrax
155
177
  redirect_to importers_path, notice: 'Importer was successfully updated.'
156
178
  end
157
179
  else
158
- if api_request?
159
- json_response('update', :unprocessable_entity, 'Something went wrong.')
160
- else
161
- render :edit
162
- end
180
+ render_update_failure
163
181
  end
182
+ rescue Bulkrax::UnsafeCloudSourceError, Bulkrax::UnsafePathError => e
183
+ reject_unsafe_cloud_files(e)
164
184
  end
165
185
  # rubocop:enable all
166
186
  # rubocop:enable Metrics/MethodLength
@@ -177,7 +197,6 @@ module Bulkrax
177
197
 
178
198
  # PUT /importers/1
179
199
  def continue
180
- @importer = Importer.find(params[:importer_id])
181
200
  params[:importer] = { name: @importer.name }
182
201
  @importer.validate_only = false
183
202
  update
@@ -185,8 +204,7 @@ module Bulkrax
185
204
 
186
205
  # GET /importer/1/upload_corrected_entries
187
206
  def upload_corrected_entries
188
- @importer = Importer.find(params[:importer_id])
189
- return unless defined?(::Hyrax)
207
+ return unless Bulkrax.hyrax_loaded?
190
208
  add_breadcrumb t(:'hyrax.controls.home'), main_app.root_path
191
209
  add_breadcrumb t(:'hyrax.dashboard.breadcrumbs.admin'), hyrax.dashboard_path
192
210
  add_breadcrumb t(:'bulkrax.headings.importers'), bulkrax.importers_path
@@ -197,7 +215,6 @@ module Bulkrax
197
215
  # POST /importer/1/upload_corrected_entries_file
198
216
  def upload_corrected_entries_file
199
217
  file = params[:importer][:parser_fields].delete(:file)
200
- @importer = Importer.find(params[:importer_id])
201
218
  if file.present?
202
219
  @importer[:parser_fields]['partial_import_file_path'] = @importer.parser.write_partial_import_file(file)
203
220
  @importer.save
@@ -242,18 +259,12 @@ module Bulkrax
242
259
 
243
260
  # GET /importers/1/export_errors
244
261
  def export_errors
245
- @importer = Importer.find(params[:importer_id])
246
262
  @importer.write_errored_entries_file
247
263
  send_content
248
264
  end
249
265
 
250
266
  private
251
267
 
252
- # Use callbacks to share common setup or constraints between actions.
253
- def set_importer
254
- @importer = Importer.find(params[:id] || params[:importer_id])
255
- end
256
-
257
268
  def importable_params
258
269
  params.except(:selected_files)
259
270
  end
@@ -267,7 +278,6 @@ module Bulkrax
267
278
  importable_params.require(:importer).permit(
268
279
  :name,
269
280
  :admin_set_id,
270
- :user_id,
271
281
  :frequency,
272
282
  :parser_klass,
273
283
  :limit,
@@ -332,6 +342,36 @@ module Bulkrax
332
342
  'text/csv'
333
343
  end
334
344
 
345
+ # Reports an unsafe cloud-file batch without leaking the error's class
346
+ # or backtrace (which could include a provider auth header) to the
347
+ # response; `e.message` is always one of our own static, credential-free
348
+ # strings (see Bulkrax::CloudSourceValidator / Bulkrax::SafePath).
349
+ def reject_unsafe_cloud_files(e)
350
+ Rails.logger.error("Bulkrax: rejected unsafe cloud file batch for importer #{@importer&.id}: #{e.class}: #{e.message}")
351
+ if api_request?
352
+ json_response('create', :unprocessable_entity, e.message)
353
+ else
354
+ flash[:error] = e.message
355
+ redirect_back fallback_location: bulkrax.importers_path
356
+ end
357
+ end
358
+
359
+ def render_create_failure
360
+ if api_request?
361
+ json_response('create', :unprocessable_entity)
362
+ else
363
+ render :new
364
+ end
365
+ end
366
+
367
+ def render_update_failure
368
+ if api_request?
369
+ json_response('update', :unprocessable_entity, 'Something went wrong.')
370
+ else
371
+ render :edit
372
+ end
373
+ end
374
+
335
375
  def render_request(message, validate_only = false)
336
376
  if api_request?
337
377
  json_response('create', :created, message)
@@ -347,6 +387,8 @@ module Bulkrax
347
387
  params[:commit] == 'Update and Harvest Updated Items'
348
388
  end
349
389
 
390
+ # @return [Boolean] the result of the save, so the caller can treat a
391
+ # failure here like an ordinary validation failure
350
392
  def set_files_parser_fields
351
393
  @importer.parser_fields['update_files'] =
352
394
  @importer.parser_fields['replace_files'] =
@@ -359,16 +401,12 @@ module Bulkrax
359
401
  elsif params[:commit] == 'Remove and Rerun'
360
402
  @importer.parser_fields['remove_and_rerun'] = true
361
403
  elsif params[:commit] == 'Update and Harvest Updated Items'
362
- return
404
+ return true
363
405
  else
364
406
  @importer.parser_fields['metadata_only'] = true
365
407
  end
366
408
  @importer.save
367
409
  end
368
-
369
- def check_permissions
370
- raise CanCan::AccessDenied unless current_ability.can_import_works?
371
- end
372
410
  end
373
411
  # rubocop:enable Metrics/ClassLength
374
412
  end
@@ -83,7 +83,7 @@ module Bulkrax
83
83
  @exporter_table_search
84
84
  end
85
85
 
86
- def format_importers(importers)
86
+ def format_importers(importers, filtered_count = Bulkrax::Importer.accessible_by(current_ability).count)
87
87
  result = importers.map do |i|
88
88
  {
89
89
  name: view_context.link_to(i.name, view_context.importer_path(i)),
@@ -102,12 +102,12 @@ module Bulkrax
102
102
  end
103
103
  {
104
104
  data: result,
105
- recordsTotal: Bulkrax::Importer.count,
106
- recordsFiltered: Bulkrax::Importer.count
105
+ recordsTotal: api_request? ? Bulkrax::Importer.count : Bulkrax::Importer.accessible_by(current_ability).count,
106
+ recordsFiltered: filtered_count
107
107
  }
108
108
  end
109
109
 
110
- def format_exporters(exporters)
110
+ def format_exporters(exporters, filtered_count = Bulkrax::Exporter.accessible_by(current_ability).count)
111
111
  result = exporters.map do |e|
112
112
  {
113
113
  name: view_context.link_to(e.name, view_context.exporter_path(e)),
@@ -119,8 +119,8 @@ module Bulkrax
119
119
  end
120
120
  {
121
121
  data: result,
122
- recordsTotal: Bulkrax::Exporter.count,
123
- recordsFiltered: Bulkrax::Exporter.count
122
+ recordsTotal: Bulkrax::Exporter.accessible_by(current_ability).count,
123
+ recordsFiltered: filtered_count
124
124
  }
125
125
  end
126
126
 
@@ -146,8 +146,10 @@ module Bulkrax
146
146
  def entry_util_links(e, item)
147
147
  links = []
148
148
  links << view_context.link_to(view_context.raw('<span class="fa fa-info-circle"></span>'), view_context.item_entry_path(item, e))
149
- links << "<a class='fa fa-repeat' data-toggle='modal' data-target='#bulkraxItemModal' data-entry-id='#{e.id}'></a>" if view_context.an_importer?(item)
150
- links << view_context.link_to(view_context.raw('<span class="fa fa-trash"></span>'), view_context.item_entry_path(item, e), method: :delete, data: { confirm: 'This will delete the entry and any work associated with it. Are you sure?' })
149
+ if can?(:update, item)
150
+ links << "<a class='fa fa-repeat' data-toggle='modal' data-target='#bulkraxItemModal' data-entry-id='#{e.id}'></a>" if view_context.an_importer?(item)
151
+ links << view_context.link_to(view_context.raw('<span class="fa fa-trash"></span>'), view_context.item_entry_path(item, e), method: :delete, data: { confirm: 'This will delete the entry and any work associated with it. Are you sure?' })
152
+ end
151
153
  links.join(" ")
152
154
  end
153
155
 
@@ -166,16 +168,20 @@ module Bulkrax
166
168
  def importer_util_links(i)
167
169
  links = []
168
170
  links << view_context.link_to(view_context.raw('<span class="fa fa-info-circle"></span>'), importer_path(i))
169
- links << view_context.link_to(view_context.raw('<span class="fa fa-pencil"></span>'), edit_importer_path(i))
170
- links << view_context.link_to(view_context.raw('<span class="fa fa-remove"></span>'), i, method: :delete, data: { confirm: 'Are you sure?' })
171
+ if can?(:update, i)
172
+ links << view_context.link_to(view_context.raw('<span class="fa fa-pencil"></span>'), edit_importer_path(i))
173
+ links << view_context.link_to(view_context.raw('<span class="fa fa-remove"></span>'), i, method: :delete, data: { confirm: 'Are you sure?' })
174
+ end
171
175
  links.join(" ")
172
176
  end
173
177
 
174
178
  def exporter_util_links(i)
175
179
  links = []
176
180
  links << view_context.link_to(view_context.raw('<span class="fa fa-info-circle"></span>'), exporter_path(i))
177
- links << view_context.link_to(view_context.raw('<span class="fa fa-pencil"></span>'), edit_exporter_path(i), data: { turbolinks: false })
178
- links << view_context.link_to(view_context.raw('<span class="fa fa-remove"></span>'), i, method: :delete, data: { confirm: 'Are you sure?' })
181
+ if can?(:update, i)
182
+ links << view_context.link_to(view_context.raw('<span class="fa fa-pencil"></span>'), edit_exporter_path(i), data: { turbolinks: false })
183
+ links << view_context.link_to(view_context.raw('<span class="fa fa-remove"></span>'), i, method: :delete, data: { confirm: 'Are you sure?' })
184
+ end
179
185
  links.join(" ")
180
186
  end
181
187
 
@@ -43,7 +43,7 @@ module Bulkrax
43
43
  end
44
44
 
45
45
  def uploaded_files_scope
46
- return [] unless defined?(::Hyrax)
46
+ return [] unless Bulkrax.hyrax_loaded?
47
47
 
48
48
  base = Hyrax::UploadedFile.where(id: params[:uploaded_files])
49
49
  if respond_to?(:current_user) && current_user.present?
@@ -139,21 +139,38 @@ module Bulkrax
139
139
  csv_by_depth[shallowest_depth]
140
140
  end
141
141
 
142
- # Persists uploaded file(s) and/or cloud files onto the importer record.
142
+ # Validates every selected cloud-file entry — raw name and source
143
+ # safety — before the importer is created or updated at all. Must not
144
+ # touch the importer's workspace (no path_for_import, no directory
145
+ # creation, no download): for #create that workspace doesn't exist
146
+ # yet, and for #update it must not be assumed safe to touch before the
147
+ # proposed attributes are known to be acceptable.
148
+ #
149
+ # @param cloud_files [Hash, nil] cloud file specs from browse-everything
150
+ # @raise [Bulkrax::UnsafePathError, Bulkrax::UnsafeCloudSourceError] on the first unsafe entry
151
+ def preflight_cloud_files!(cloud_files)
152
+ return if cloud_files.blank?
153
+
154
+ cloud_files.values.each { |file| Bulkrax::CloudSourceValidator.preflight!(file) }
155
+ end
156
+
157
+ # Assigns files/cloud selections onto the importer record's in-memory
158
+ # parser fields, but does not save: the caller (ImportersController)
159
+ # owns the second save, so it can check its result and avoid rendering
160
+ # success or enqueueing an import on failure.
161
+ #
143
162
  # @param file [ActionDispatch::Http::UploadedFile, nil] a directly uploaded file
144
163
  # @param cloud_files [Hash, nil] cloud file paths from browse-everything
145
164
  # @param uploads [ActiveRecord::Relation, Array, nil] Hyrax::UploadedFile records
165
+ # @return [Boolean] true if any parser field was touched and a second save is needed
146
166
  def files_for_import(file, cloud_files, uploads)
147
- return if file.blank? && cloud_files.blank? && uploads.blank?
167
+ @cloud_download_plan = nil
168
+ return false if file.blank? && cloud_files.blank? && uploads.blank?
148
169
 
149
170
  @importer[:parser_fields]['import_file_path'] = @importer.parser.write_import_file(file) if file.present?
150
171
  if cloud_files.present?
151
172
  @importer[:parser_fields]['cloud_file_paths'] = cloud_files
152
- # For BagIt, there will only be one bag, so we get the file_path back and set import_file_path
153
- # For CSV, we expect only file uploads, so we won't get the file_path back
154
- # and we expect the import_file_path to be set already
155
- target = @importer.parser.retrieve_cloud_files(cloud_files, @importer)
156
- @importer[:parser_fields]['import_file_path'] = target if target.present?
173
+ assign_cloud_file_paths(cloud_files)
157
174
  end
158
175
 
159
176
  if uploads.present?
@@ -162,7 +179,36 @@ module Bulkrax
162
179
  end
163
180
  end
164
181
 
165
- @importer.save
182
+ true
183
+ end
184
+
185
+ # For a parser that separates planning from dispatch (currently only
186
+ # CsvParser), resolves and validates the whole batch and records the
187
+ # target paths, deferring enqueue until {#dispatch_cloud_downloads!} is
188
+ # called after the controller persists those paths. Other parsers
189
+ # (e.g. BagitParser) already complete cloud retrieval synchronously
190
+ # and return the resolved path directly, so there's nothing left to
191
+ # dispatch.
192
+ def assign_cloud_file_paths(cloud_files)
193
+ parser = @importer.parser
194
+ if parser.respond_to?(:plan_cloud_files)
195
+ @cloud_download_plan = parser.plan_cloud_files(cloud_files)
196
+ @importer[:parser_fields]['original_file_paths'] =
197
+ @cloud_download_plan.map { |_file, _relative_filename, target_file| target_file }
198
+ else
199
+ target = parser.retrieve_cloud_files(cloud_files, @importer)
200
+ @importer[:parser_fields]['import_file_path'] = target if target.present?
201
+ end
202
+ end
203
+
204
+ # Dispatches a plan prepared by {#assign_cloud_file_paths}, if any.
205
+ # Called by the controller only after it has successfully persisted
206
+ # the resolved paths, so a download is never enqueued for a path the
207
+ # importer doesn't actually have on record.
208
+ def dispatch_cloud_downloads!
209
+ return if @cloud_download_plan.blank?
210
+
211
+ @importer.parser.dispatch_cloud_files!(@cloud_download_plan, @importer)
166
212
  end
167
213
 
168
214
  def write_files(files)
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Bulkrax
4
+ # Raised by {Bulkrax::CloudSourceValidator} when a cloud-file source
5
+ # (the `url` a user selected via browse-everything, or supplied directly
6
+ # in importer params) can't be safely downloaded: a scheme other than
7
+ # `file`/`http`/`https`, a `file:` source outside the configured
8
+ # filesystem-provider root, or an HTTP(S) source that resolves to a
9
+ # private, loopback, or link-local/metadata address.
10
+ class UnsafeCloudSourceError < StandardError; end
11
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Bulkrax
4
+ # Raised by {Bulkrax::SafePath} when a path would escape its configured
5
+ # root, whether that root is a ZIP extraction directory, an importer's
6
+ # trusted workspace, or a configured filesystem-provider home.
7
+ class UnsafePathError < StandardError; end
8
+ end
@@ -168,8 +168,8 @@ module Bulkrax
168
168
  end
169
169
 
170
170
  def self.solr_name(field_name)
171
- if defined?(Hyrax)
172
- Hyrax.index_field_mapper.solr_name(field_name)
171
+ if defined?(Hyrax) && Hyrax.respond_to?(:config) && Hyrax.config.respond_to?(:index_field_mapper)
172
+ Hyrax.config.index_field_mapper.solr_name(field_name)
173
173
  else
174
174
  ActiveFedora.index_field_mapper.solr_name(field_name)
175
175
  end
@@ -280,11 +280,14 @@ module Bulkrax
280
280
  # @param admin_set_id [String, nil] admin set used to resolve contexts
281
281
  # @return [Array<String>]
282
282
  def self.schema_properties(klass:, admin_set_id: nil)
283
- cached_schema_for(klass: klass, admin_set_id: admin_set_id).map { |k| k.name.to_s }
283
+ entry = schema_cache_entry(klass: klass, admin_set_id: admin_set_id)
284
+ entry[:properties] ||= entry[:schema].map { |k| k.name.to_s }.freeze
284
285
  end
285
286
 
286
287
  ##
287
- # Returns the schema for a model, memoized per (klass, admin_set_id) pair.
288
+ # Returns the schema for a model, memoized per (tenant, klass, admin_set_id)
289
+ # for the current request or job only, so a new metadata profile or a
290
+ # change to an admin set's contexts is picked up without a restart.
288
291
  # Delegates to +Hyrax.schema_for+ when available so that context-gated
289
292
  # properties are included without Bulkrax knowing about flexibility internals.
290
293
  #
@@ -292,16 +295,25 @@ module Bulkrax
292
295
  # @param admin_set_id [String, nil]
293
296
  # @return [Dry::Types::Hash]
294
297
  def self.cached_schema_for(klass:, admin_set_id: nil)
295
- @cached_schema_map ||= {}
296
- key = [klass.name, admin_set_id].compact.join('|')
297
- @cached_schema_map[key] ||=
298
- if admin_set_id.present? && defined?(Hyrax) && Hyrax.respond_to?(:schema_for)
299
- Hyrax.schema_for(klass: klass, admin_set_id: admin_set_id)
300
- else
301
- klass.new.singleton_class.schema || klass.schema
302
- end
298
+ schema_cache_entry(klass: klass, admin_set_id: admin_set_id)[:schema]
303
299
  end
304
300
 
301
+ def self.schema_cache_entry(klass:, admin_set_id:)
302
+ tenant = defined?(Apartment::Tenant) ? Apartment::Tenant.current : 'single'
303
+ Bulkrax::Current.schema_cache ||= {}
304
+ Bulkrax::Current.schema_cache[[tenant, klass, admin_set_id]] ||= { schema: resolve_schema(klass: klass, admin_set_id: admin_set_id) }
305
+ end
306
+ private_class_method :schema_cache_entry
307
+
308
+ def self.resolve_schema(klass:, admin_set_id:)
309
+ if admin_set_id.present? && defined?(Hyrax) && Hyrax.respond_to?(:schema_for)
310
+ Hyrax.schema_for(klass: klass, admin_set_id: admin_set_id)
311
+ else
312
+ klass.new.singleton_class.schema || klass.schema
313
+ end
314
+ end
315
+ private_class_method :resolve_schema
316
+
305
317
  def self.ordered_file_sets_for(object)
306
318
  return [] if object.blank?
307
319
 
@@ -434,7 +446,7 @@ module Bulkrax
434
446
 
435
447
  file_attrs = []
436
448
  uploads.each_with_index do |f, index|
437
- file_attrs << ({ uploaded_file_id: f["id"].to_s, filename: files[index]["file_name"] }).merge(additional_attributes[index])
449
+ file_attrs << { uploaded_file_id: f["id"].to_s, filename: files[index]["file_name"] }.merge(additional_attributes[index])
438
450
  end
439
451
  file_attrs.compact.uniq
440
452
  end
@@ -501,7 +513,7 @@ module Bulkrax
501
513
  @permitted_attributes ||= begin
502
514
  bare = base_permitted_attributes + if klass.respond_to?(:schema)
503
515
  admin_set_id = attributes[:admin_set_id] || attributes['admin_set_id']
504
- Bulkrax::ValkyrieObjectFactory.schema_properties(klass: klass, admin_set_id: admin_set_id)
516
+ Bulkrax::ValkyrieObjectFactory.schema_properties(klass: klass, admin_set_id: admin_set_id).map(&:to_sym)
505
517
  else
506
518
  klass.properties.keys.map(&:to_sym)
507
519
  end
@@ -590,18 +602,18 @@ module Bulkrax
590
602
 
591
603
  # @Override Destroy existing files with Hyrax::Transactions
592
604
  def destroy_existing_files(object: @object)
593
- existing_files = Hyrax.custom_queries.find_child_file_sets(resource: object)
594
- return if existing_files.empty?
595
-
596
- existing_files.each do |fs|
605
+ destroyed_ids = []
606
+ Hyrax.custom_queries.find_child_file_sets(resource: object).each do |fs|
597
607
  transactions["file_set.destroy"]
598
608
  .with_step_args("file_set.remove_from_work" => { user: @user },
599
609
  "file_set.delete" => { user: @user })
600
610
  .call(fs)
601
611
  .value!
612
+ destroyed_ids << fs.id
602
613
  end
614
+ return if destroyed_ids.empty?
603
615
 
604
- object.member_ids = object.member_ids.reject { |m| existing_files.detect { |f| f.id == m } }
616
+ object.member_ids -= destroyed_ids
605
617
  object.rendering_ids = []
606
618
  object.representative_id = nil
607
619
  object.thumbnail_id = nil
@@ -17,7 +17,7 @@ module Bulkrax
17
17
  end
18
18
 
19
19
  def check_admin_set
20
- return unless defined?(::Hyrax)
20
+ return unless Bulkrax.hyrax_loaded?
21
21
 
22
22
  if params[:importer][:admin_set_id].blank?
23
23
  params[:importer][:admin_set_id] = Bulkrax.object_factory.default_admin_set_id
@@ -114,7 +114,7 @@ module Bulkrax
114
114
  # We can use Hyrax's lock manager when we have one available.
115
115
  # However it's not certain that this is actually working, so to be
116
116
  # as safe as possible, we will reload resources before we update.
117
- if defined?(::Hyrax)
117
+ if Bulkrax.hyrax_loaded?
118
118
  include Hyrax::Lockable
119
119
 
120
120
  def conditionally_acquire_lock_for(*args, &block)