bulkrax 8.2.3 → 8.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: efd4c183d72156bd32a32eb0e160485f1dc46c67630f6835db761bb93ab3b16b
4
- data.tar.gz: 05c58dccdae7e285db479602490dd3be7429405f931a18aede0accbd4eec3503
3
+ metadata.gz: 921dc741e247f2b7ebd31f9e1b00d95f7a5da6f42fc4106979bc2456ce52d39c
4
+ data.tar.gz: 9d595dddab1708878df613bc2b40315974af950273eb26fffbbfa6dcdd5e5958
5
5
  SHA512:
6
- metadata.gz: e64eecf68566f81c2d105640b71b8b6834e4c32fc18a416f0c026883387a257f913ad82ad676826b6a744a352edbce4a21aa82a304b23f2583f2c15a97f72078
7
- data.tar.gz: 9c4ce4f1b4d36c5dcb031ce0a8ecb6617c0114dbc891384d3de5fe6e5ee42b04761001d269d22d6b54c616aaf3cf793e461187d0e17fd20c3a025a907c20fb4e
6
+ metadata.gz: cc708da6667febae646836c3720b2a74e7b0eb6b4c0414b53a583ff54006be82a567906029d951c8172497df8538aaa25e964bd012beee3ade6d59df379cb12f
7
+ data.tar.gz: c6920e180a2eb093bbec5e29c508c124c57c77f5039c8352c69a1704169c5540e55276ad2d45429e0056ea36949b2dca7155da375d6da443b3e99608c1cc1e6e
@@ -78,6 +78,7 @@ module Bulkrax
78
78
 
79
79
  # POST /importers
80
80
  # rubocop:disable Metrics/MethodLength
81
+ # rubocop:disable Metrics/AbcSize
81
82
  def create
82
83
  # rubocop:disable Style/IfInsideElse
83
84
  if api_request?
@@ -92,8 +93,15 @@ module Bulkrax
92
93
  # the following line is needed to handle updating remote files of a FileSet
93
94
  # on a new import otherwise it only gets updated during the update path
94
95
  @importer.parser_fields['update_files'] = true if params[:commit] == 'Create and Import'
96
+
97
+ # Reject an unsafe cloud selection before the importer is ever
98
+ # persisted, rather than after save when a row would already exist.
99
+ preflight_cloud_files!(cloud_files)
100
+
95
101
  if @importer.save
96
- files_for_import(file, cloud_files)
102
+ return render_create_failure if files_for_import(file, cloud_files) && !@importer.save
103
+ dispatch_cloud_downloads!
104
+
97
105
  if params[:commit] == 'Create and Import'
98
106
  Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id)
99
107
  render_request('Importer was successfully created and import has been queued.')
@@ -104,14 +112,13 @@ module Bulkrax
104
112
  render_request('Importer was successfully created.')
105
113
  end
106
114
  else
107
- if api_request?
108
- json_response('create', :unprocessable_entity)
109
- else
110
- render :new
111
- end
115
+ render_create_failure
112
116
  end
113
117
  # rubocop:enable Style/IfInsideElse
118
+ rescue Bulkrax::UnsafeCloudSourceError, Bulkrax::UnsafePathError => e
119
+ reject_unsafe_cloud_files(e)
114
120
  end
121
+ # rubocop:enable Metrics/AbcSize
115
122
 
116
123
  # PATCH/PUT /importers/1
117
124
  # # @todo refactor so as to not need to disable rubocop
@@ -127,11 +134,19 @@ module Bulkrax
127
134
  # Skipped during a continue
128
135
  field_mapping_params if params[:importer][:parser_fields].present?
129
136
 
130
- if @importer.update(importer_params)
131
- files_for_import(file, cloud_files) unless file.nil? && cloud_files.nil?
137
+ # Assign the proposed attributes in memory first, so an unsafe cloud
138
+ # selection is rejected before anything is persisted; the rescue
139
+ # below then leaves the importer's persisted attributes untouched.
140
+ @importer.assign_attributes(importer_params)
141
+ preflight_cloud_files!(cloud_files)
142
+
143
+ if @importer.save
144
+ return render_update_failure if files_for_import(file, cloud_files) && !@importer.save
145
+ return render_update_failure if params[:commit] != 'Update Importer' && !set_files_parser_fields
146
+ dispatch_cloud_downloads!
147
+
132
148
  # do not perform the import
133
149
  unless params[:commit] == 'Update Importer'
134
- set_files_parser_fields
135
150
  Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id, update_harvest)
136
151
  end
137
152
  if api_request?
@@ -140,12 +155,10 @@ module Bulkrax
140
155
  redirect_to importers_path, notice: 'Importer was successfully updated.'
141
156
  end
142
157
  else
143
- if api_request?
144
- json_response('update', :unprocessable_entity, 'Something went wrong.')
145
- else
146
- render :edit
147
- end
158
+ render_update_failure
148
159
  end
160
+ rescue Bulkrax::UnsafeCloudSourceError, Bulkrax::UnsafePathError => e
161
+ reject_unsafe_cloud_files(e)
149
162
  end
150
163
  # rubocop:enable all
151
164
  # rubocop:enable Metrics/MethodLength
@@ -218,18 +231,68 @@ module Bulkrax
218
231
 
219
232
  private
220
233
 
234
+ # Validates every selected cloud-file entry — raw name and source
235
+ # safety — before the importer is created or updated at all. Must not
236
+ # touch the importer's workspace (no path_for_import, no directory
237
+ # creation, no download): for #create that workspace doesn't exist
238
+ # yet, and for #update it must not be assumed safe to touch before the
239
+ # proposed attributes are known to be acceptable.
240
+ #
241
+ # @param cloud_files [Hash, nil] cloud file specs from browse-everything
242
+ # @raise [Bulkrax::UnsafePathError, Bulkrax::UnsafeCloudSourceError] on the first unsafe entry
243
+ def preflight_cloud_files!(cloud_files)
244
+ return if cloud_files.blank?
245
+
246
+ cloud_files.values.each { |file| Bulkrax::CloudSourceValidator.preflight!(file) }
247
+ end
248
+
249
+ # Assigns files/cloud selections onto the importer record's in-memory
250
+ # parser fields, but does not save: the caller owns the second save,
251
+ # so it can check its result and avoid rendering success or enqueueing
252
+ # an import on failure.
253
+ #
254
+ # @return [Boolean] true if any parser field was touched and a second save is needed
221
255
  def files_for_import(file, cloud_files)
222
- return if file.blank? && cloud_files.blank?
256
+ @cloud_download_plan = nil
257
+ return false if file.blank? && cloud_files.blank?
258
+
223
259
  @importer[:parser_fields]['import_file_path'] = @importer.parser.write_import_file(file) if file.present?
224
260
  if cloud_files.present?
225
261
  @importer[:parser_fields]['cloud_file_paths'] = cloud_files
262
+ assign_cloud_file_paths(cloud_files)
263
+ end
264
+
265
+ true
266
+ end
267
+
268
+ # For a parser that separates planning from dispatch (currently only
269
+ # CsvParser), resolves and validates the whole batch and records the
270
+ # target paths, deferring enqueue until {#dispatch_cloud_downloads!} is
271
+ # called after the controller persists those paths. Other parsers
272
+ # (e.g. BagitParser) already complete cloud retrieval synchronously
273
+ # and return the resolved path directly, so there's nothing left to
274
+ # dispatch.
275
+ def assign_cloud_file_paths(cloud_files)
276
+ parser = @importer.parser
277
+ if parser.respond_to?(:plan_cloud_files)
278
+ @cloud_download_plan = parser.plan_cloud_files(cloud_files)
279
+ @importer[:parser_fields]['original_file_paths'] =
280
+ @cloud_download_plan.map { |_file, _relative_filename, target_file| target_file }
281
+ else
226
282
  # For BagIt, there will only be one bag, so we get the file_path back and set import_file_path
227
- # For CSV, we expect only file uploads, so we won't get the file_path back
228
- # and we expect the import_file_path to be set already
229
- target = @importer.parser.retrieve_cloud_files(cloud_files, @importer)
283
+ target = parser.retrieve_cloud_files(cloud_files, @importer)
230
284
  @importer[:parser_fields]['import_file_path'] = target if target.present?
231
285
  end
232
- @importer.save
286
+ end
287
+
288
+ # Dispatches a plan prepared by {#assign_cloud_file_paths}, if any.
289
+ # Called by the controller only after it has successfully persisted
290
+ # the resolved paths, so a download is never enqueued for a path the
291
+ # importer doesn't actually have on record.
292
+ def dispatch_cloud_downloads!
293
+ return if @cloud_download_plan.blank?
294
+
295
+ @importer.parser.dispatch_cloud_files!(@cloud_download_plan, @importer)
233
296
  end
234
297
 
235
298
  # Use callbacks to share common setup or constraints between actions.
@@ -315,6 +378,36 @@ module Bulkrax
315
378
  'text/csv'
316
379
  end
317
380
 
381
+ # Reports an unsafe cloud-file batch without leaking the error's class
382
+ # or backtrace (which could include a provider auth header) to the
383
+ # response; `e.message` is always one of our own static, credential-free
384
+ # strings (see Bulkrax::CloudSourceValidator / Bulkrax::SafePath).
385
+ def reject_unsafe_cloud_files(e)
386
+ Rails.logger.error("Bulkrax: rejected unsafe cloud file batch for importer #{@importer&.id}: #{e.class}: #{e.message}")
387
+ if api_request?
388
+ json_response('create', :unprocessable_entity, e.message)
389
+ else
390
+ flash[:error] = e.message
391
+ redirect_back fallback_location: importers_path
392
+ end
393
+ end
394
+
395
+ def render_create_failure
396
+ if api_request?
397
+ json_response('create', :unprocessable_entity)
398
+ else
399
+ render :new
400
+ end
401
+ end
402
+
403
+ def render_update_failure
404
+ if api_request?
405
+ json_response('update', :unprocessable_entity, 'Something went wrong.')
406
+ else
407
+ render :edit
408
+ end
409
+ end
410
+
318
411
  def render_request(message, validate_only = false)
319
412
  if api_request?
320
413
  json_response('create', :created, message)
@@ -330,6 +423,8 @@ module Bulkrax
330
423
  params[:commit] == 'Update and Harvest Updated Items'
331
424
  end
332
425
 
426
+ # @return [Boolean] the result of the save, so the caller can treat a
427
+ # failure here like an ordinary validation failure
333
428
  def set_files_parser_fields
334
429
  @importer.parser_fields['update_files'] =
335
430
  @importer.parser_fields['replace_files'] =
@@ -342,7 +437,7 @@ module Bulkrax
342
437
  elsif params[:commit] == 'Remove and Rerun'
343
438
  @importer.parser_fields['remove_and_rerun'] = true
344
439
  elsif params[:commit] == 'Update and Harvest Updated Items'
345
- return
440
+ return true
346
441
  else
347
442
  @importer.parser_fields['metadata_only'] = true
348
443
  end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Bulkrax
4
+ # Raised by {Bulkrax::CloudSourceValidator} when a cloud-file source
5
+ # (the `url` a user selected via browse-everything, or supplied directly
6
+ # in importer params) can't be safely downloaded: a scheme other than
7
+ # `file`/`http`/`https`, a `file:` source outside the configured
8
+ # filesystem-provider root, or an HTTP(S) source that resolves to a
9
+ # private, loopback, or link-local/metadata address.
10
+ class UnsafeCloudSourceError < StandardError; end
11
+ end
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Bulkrax
4
+ # Raised by {Bulkrax::SafePath} when a path would escape its configured
5
+ # root, whether that root is a ZIP extraction directory, an importer's
6
+ # trusted workspace, or a configured filesystem-provider home.
7
+ class UnsafePathError < StandardError; end
8
+ end
@@ -1,31 +1,166 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  module Bulkrax
3
4
  class DownloadCloudFileJob < ApplicationJob
4
5
  queue_as Bulkrax.config.ingest_queue_name
5
6
 
6
- include ActionView::Helpers::NumberHelper
7
-
8
- # Retrieve cloud file and write to the imports directory
9
- # Note: if using the file system, the mounted directory in
7
+ # Retrieve a cloud file and write it to the importer's workspace.
8
+ #
9
+ # `importer_id:` and `relative_filename:` are required keywords: the
10
+ # destination is always recomputed here, confined to this importer's
11
+ # own workspace, rather than trusted from a serialized job argument.
12
+ # `target_file` is a deprecated positional placeholder, kept only so
13
+ # that a pre-upgrade call shape (`file`, `target_file`) still binds
14
+ # without Ruby raising its own ArgumentError — it is never read, even
15
+ # when supplied alongside otherwise-valid keywords, so it can't be
16
+ # used to redirect the destination. A call made this way is missing
17
+ # the required keywords, so it raises below rather than running.
18
+ #
19
+ # Note: if using the file system provider, the mounted directory in
10
20
  # browse_everything MUST be shared by web and worker servers
11
- def perform(file, target_file)
12
- retriever = BrowseEverything::Retriever.new
13
- last_logged_time = Time.zone.now
14
- log_interval = 3.seconds
15
-
16
- retriever.download(file, target_file) do |filename, retrieved, total|
17
- percentage = (retrieved.to_f / total.to_f) * 100
18
- current_time = Time.zone.now
19
-
20
- if (current_time - last_logged_time) >= log_interval
21
- # Use number_to_human_size for formatting
22
- readable_retrieved = number_to_human_size(retrieved)
23
- readable_total = number_to_human_size(total)
24
- Rails.logger.info "Downloaded #{readable_retrieved} of #{readable_total}, #{filename}: #{percentage.round}% complete"
25
- last_logged_time = current_time
26
- end
21
+ def perform(file, target_file = nil, importer_id: nil, relative_filename: nil)
22
+ raise Bulkrax::UnsafePathError, I18n.t('bulkrax.importer.browse_everything.errors.missing_destination_context') if importer_id.blank? || relative_filename.blank?
23
+
24
+ # `file` and `relative_filename` are independent arguments and may
25
+ # not agree (a tampered or replayed job call could supply either
26
+ # one separately) — check the raw file_name directly rather than
27
+ # assuming it was already validated by whatever enqueued this job.
28
+ Bulkrax::SafePath.reject_unsafe_entry!(file['file_name'])
29
+
30
+ importer = Bulkrax::Importer.find(importer_id)
31
+ target_file = Bulkrax::SafePath.confine!(importer.parser.path_for_import, relative_filename, require_file: true)
32
+
33
+ # Re-validate the source here too: this job may run on a different
34
+ # process/host than the one that enqueued it, and could be invoked
35
+ # directly with a tampered or stale `file` argument.
36
+ Bulkrax::CloudSourceValidator.validate!(file)
37
+
38
+ download(file, target_file)
39
+ Rails.logger.info "Bulkrax: cloud file download complete for #{file['file_name']}"
40
+ end
41
+
42
+ private
43
+
44
+ # Downloads through a temp file in the *same* directory as
45
+ # `target_file` (so the final move is same-filesystem and atomic),
46
+ # named with a random suffix rather than anything derived from
47
+ # caller-controlled input, and removed on any failure. This means a
48
+ # failed or non-success download can never truncate or corrupt an
49
+ # already-good destination file from a previous successful import.
50
+ def download(file, target_file)
51
+ temp_file = "#{target_file}.#{SecureRandom.hex(8)}.part"
52
+ uri = Addressable::URI.parse(file['url'].to_s)
53
+ if uri.scheme == 'file'
54
+ download_file_source(uri, temp_file)
55
+ else
56
+ download_http_source(file, URI.parse(file['url'].to_s), temp_file)
57
+ end
58
+ FileUtils.mv(temp_file, target_file)
59
+ ensure
60
+ FileUtils.rm_f(temp_file) if temp_file && File.exist?(temp_file)
61
+ end
62
+
63
+ # The source was already confined to the configured filesystem
64
+ # provider's root by {Bulkrax::CloudSourceValidator}.
65
+ def download_file_source(uri, temp_file)
66
+ source_path = URI::DEFAULT_PARSER.unescape(uri.path.to_s)
67
+ IO.copy_stream(source_path, temp_file)
68
+ end
69
+
70
+ # Resolved, IP-pinned, redirect-free, and streamed straight to disk in
71
+ # bounded-size chunks — rather than handing the URL back to a transport
72
+ # (browse-everything's Retriever) that re-resolves the hostname with no
73
+ # SSRF protection at all. This is a *separate* connection from the one
74
+ # {Bulkrax::CloudSourceValidator}'s HEAD check made; each independently
75
+ # applies the same validation at its own connection time, so there is
76
+ # no single "pinned connection" shared between validation and download.
77
+ #
78
+ # ssrf_filter 1.0.8's public `get`/`head` API has no response-streaming
79
+ # hook: `fetch_once` calls `Net::HTTP#request` with no block, which
80
+ # reads the entire response body into memory before returning; the
81
+ # block `get`/`head` accept is for customizing the *outgoing* request,
82
+ # not consuming the response. So we can't use `SsrfFilter.get` here
83
+ # without buffering the whole remote file in memory first.
84
+ #
85
+ # Reuse ssrf_filter's resolver and IP policy, then pin the validated
86
+ # address with Net::HTTP#ipaddr=. Keeping the original hostname on
87
+ # the HTTP connection preserves native TLS SNI and certificate
88
+ # verification without relying on ssrf_filter's TLS monkeypatches.
89
+ def download_http_source(file, uri, temp_file)
90
+ headers = Bulkrax::CloudSourceValidator.request_headers(file)
91
+ stream_download(uri, headers) { |response| handle_response(file, response, temp_file) }
92
+ rescue SsrfFilter::Error
93
+ # The validator's own HEAD check already ran, but DNS can change
94
+ # between that check and this GET — same rationale as
95
+ # Bulkrax::CloudSourceValidator#validate_http_scheme!, and the same
96
+ # static, class-agnostic message for the same reason.
97
+ raise Bulkrax::UnsafeCloudSourceError, 'blocked unsafe remote source'
98
+ end
99
+
100
+ def handle_response(file, response, temp_file)
101
+ unless response.is_a?(Net::HTTPSuccess)
102
+ # Static on purpose — this becomes the importer's status message,
103
+ # not just an ops log; see Bulkrax::CloudSourceValidator for why
104
+ # status codes/response details aren't echoed into it. The status
105
+ # code itself is logged (not attacker-controlled, not a credential).
106
+ Rails.logger.error("Bulkrax: cloud file download failed for #{file['file_name']} with status #{response.code}")
107
+ raise Bulkrax::UnsafeCloudSourceError, 'download failed: non-success response'
108
+ end
109
+
110
+ File.open(temp_file, 'wb') { |io| response.read_body { |chunk| io.write(chunk) } }
111
+ end
112
+
113
+ # Resolves `uri`'s host, rejects it unless it has at least one public
114
+ # IP (reusing ssrf_filter's own resolver/blacklist — see above), and
115
+ # connects directly to that IP: with the environment proxy explicitly
116
+ # disabled (`Net::HTTP.new`'s `p_addr` defaults to `:ENV`, which would
117
+ # otherwise route through `http_proxy`/`https_proxy` and hand the
118
+ # pinned connection to a third party), and with the real hostname sent
119
+ # as the `Host` header and used for TLS SNI/certificate verification
120
+ # instead of the IP. Yields the live `Net::HTTPResponse` to the block
121
+ # *before* its body is read, so the caller can stream it.
122
+ def stream_download(uri, headers)
123
+ raise SsrfFilter::InvalidUriScheme, "URI scheme '#{uri.scheme}' not in whitelist" unless
124
+ SsrfFilter::DEFAULT_SCHEME_WHITELIST.include?(uri.scheme)
125
+
126
+ hostname = uri.hostname
127
+ ip_addresses = SsrfFilter::DEFAULT_RESOLVER.call(hostname)
128
+ raise SsrfFilter::UnresolvedHostname, "Could not resolve hostname '#{hostname}'" if ip_addresses.empty?
129
+
130
+ public_addresses = ip_addresses.reject { |ip| unsafe_ip_address?(ip) }
131
+ raise SsrfFilter::PrivateIPAddress, "Hostname '#{hostname}' has no public ip addresses" if public_addresses.empty?
132
+
133
+ request = Net::HTTP::Get.new(uri)
134
+ request['host'] = host_header(hostname, uri)
135
+ headers.each { |key, value| request[key] = value }
136
+
137
+ http = Net::HTTP.new(hostname, uri.port, nil)
138
+ http.ipaddr = public_addresses.sample.to_s
139
+ http.use_ssl = (uri.scheme == 'https')
140
+ http.start { |connection| connection.request(request) { |response| yield response } }
141
+ end
142
+
143
+ # Mirrors `SsrfFilter.unsafe_ip_address?`, but against its public
144
+ # `IPV4_BLACKLIST`/`IPV6_BLACKLIST` constants directly rather than
145
+ # calling that private method — we only ever check single resolved
146
+ # addresses here (never a CIDR range), so the masked-range guard it
147
+ # also carries doesn't apply.
148
+ def unsafe_ip_address?(ip_address)
149
+ return SsrfFilter::IPV4_BLACKLIST.any? { |range| range.include?(ip_address) } if ip_address.ipv4?
150
+ return SsrfFilter::IPV6_BLACKLIST.any? { |range| range.include?(ip_address) } if ip_address.ipv6?
151
+
152
+ true
153
+ end
154
+
155
+ # Matches `SsrfFilter.host_header`: the `Host` header should carry the
156
+ # real hostname (not the pinned IP we actually connect to), with the
157
+ # port suffixed only when it isn't the scheme's default.
158
+ def host_header(hostname, uri)
159
+ if (uri.port == 80 && uri.scheme == 'http') || (uri.port == 443 && uri.scheme == 'https')
160
+ hostname
161
+ else
162
+ "#{hostname}:#{uri.port}"
27
163
  end
28
- Rails.logger.info "Download complete: #{file['url']} to #{target_file}"
29
164
  end
30
165
  end
31
166
  end
@@ -430,15 +430,38 @@ module Bulkrax
430
430
  def unzip(file_to_unzip)
431
431
  return untar(file_to_unzip) if file_to_unzip.end_with?('.tar.gz')
432
432
 
433
+ dest_dir = importer_unzip_path
433
434
  Zip::File.open(file_to_unzip) do |zip_file|
434
435
  zip_file.each do |entry|
435
- entry_path = File.join(importer_unzip_path, entry.name)
436
+ # Zip Slip: reject/confine via the same Bulkrax::SafePath used
437
+ # for cloud-file destinations, rather than a bare File.join of
438
+ # attacker-controlled entry.name.
439
+ entry_path = Bulkrax::SafePath.confine!(dest_dir, entry.name)
436
440
  FileUtils.mkdir_p(File.dirname(entry_path))
437
- zip_file.extract(entry, entry_path) unless File.exist?(entry_path)
441
+ next if File.exist?(entry_path)
442
+
443
+ extract_zip_entry(zip_file, entry, dest_dir, entry.name, entry_path)
438
444
  end
439
445
  end
440
446
  end
441
447
 
448
+ # rubyzip 2.x: extract(entry, absolute_dest_path)
449
+ # rubyzip 3.x: extract(entry, relative_name, destination_directory: dir)
450
+ #
451
+ # entry_path (from Bulkrax::SafePath.confine!) is always an absolute
452
+ # path; rubyzip 3.x's 2-positional-arg #extract treats its second
453
+ # argument as a name relative to the current directory rather than a
454
+ # destination path, which silently re-joins an already-absolute path
455
+ # onto the cwd. The keyword form is the only one both versions can be
456
+ # trusted to interpret as "this absolute/relative destination".
457
+ def extract_zip_entry(zip_file, entry, dest_dir, relative_name, dest_path)
458
+ if zip_file.method(:extract).arity == 2
459
+ zip_file.extract(entry, dest_path)
460
+ else
461
+ zip_file.extract(entry, relative_name, destination_directory: dest_dir)
462
+ end
463
+ end
464
+
442
465
  def untar(file_to_untar)
443
466
  Dir.mkdir(importer_unzip_path) unless File.directory?(importer_unzip_path)
444
467
  command = "tar -xzf #{Shellwords.escape(file_to_untar)} -C #{Shellwords.escape(importer_unzip_path)}"