bulkrax 9.5.1 → 9.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/controllers/bulkrax/importers_controller.rb +58 -17
- data/app/controllers/concerns/bulkrax/importer_file_handler.rb +54 -8
- data/app/errors/bulkrax/unsafe_cloud_source_error.rb +11 -0
- data/app/errors/bulkrax/unsafe_path_error.rb +8 -0
- data/app/jobs/bulkrax/download_cloud_file_job.rb +156 -21
- data/app/parsers/bulkrax/application_parser.rb +10 -10
- data/app/parsers/bulkrax/application_parser.rb.orig +545 -0
- data/app/parsers/bulkrax/bagit_parser.rb +10 -3
- data/app/parsers/bulkrax/csv_parser.rb +46 -15
- data/app/services/bulkrax/cloud_source_validator.rb +123 -0
- data/app/services/bulkrax/safe_path.rb +129 -0
- data/config/locales/bulkrax.en.yml +2 -0
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +1 -0
- metadata +28 -7
- data/app/factories/bulkrax/valkyrize-hyku.code-workspace +0 -19
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: bc34b29d0f929a95964a15424a6127987da2c13d143c0a7f249c681b4f8f195d
|
|
4
|
+
data.tar.gz: e7aad35ef43d7dea75a9dade1905bab4f02a965c2fae8fb34ae265fabd242504
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7a4ec7d51d867f82044c9dc8be3fb3de9c9104bdc147b7418df80cd3ce69eb7d7a7994e35c63a7e495a4b352660fafa53a57e10e8aef2366da4cea48699621a2
|
|
7
|
+
data.tar.gz: ddaa25bd902c97e6cb95e2b98b5c3ceb5321ee10eadc2032f967342242b5aee483fbac3c405f2ce167250c19623b7789c0e29575a052e37c2d67e25d536c41f3
|
|
@@ -92,7 +92,6 @@ module Bulkrax
|
|
|
92
92
|
# rubocop:disable Metrics/MethodLength
|
|
93
93
|
# rubocop:disable Metrics/AbcSize
|
|
94
94
|
def create
|
|
95
|
-
# rubocop:disable Style/IfInsideElse
|
|
96
95
|
if api_request?
|
|
97
96
|
return return_json_response unless valid_create_params?
|
|
98
97
|
end
|
|
@@ -106,8 +105,15 @@ module Bulkrax
|
|
|
106
105
|
# the following line is needed to handle updating remote files of a FileSet
|
|
107
106
|
# on a new import otherwise it only gets updated during the update path
|
|
108
107
|
@importer.parser_fields['update_files'] = true if params[:commit] == 'Create and Import'
|
|
108
|
+
|
|
109
|
+
# Reject an unsafe cloud selection before the importer is ever
|
|
110
|
+
# persisted, rather than after save when a row would already exist.
|
|
111
|
+
preflight_cloud_files!(cloud_files)
|
|
112
|
+
|
|
109
113
|
if @importer.save
|
|
110
|
-
files_for_import(file, cloud_files, uploads)
|
|
114
|
+
return render_create_failure if files_for_import(file, cloud_files, uploads) && !@importer.save
|
|
115
|
+
dispatch_cloud_downloads!
|
|
116
|
+
|
|
111
117
|
if params[:commit] == 'Create and Import'
|
|
112
118
|
Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id)
|
|
113
119
|
render_request('Importer was successfully created and import has been queued.')
|
|
@@ -118,13 +124,10 @@ module Bulkrax
|
|
|
118
124
|
render_request('Importer was successfully created.')
|
|
119
125
|
end
|
|
120
126
|
else
|
|
121
|
-
|
|
122
|
-
json_response('create', :unprocessable_entity)
|
|
123
|
-
else
|
|
124
|
-
render :new
|
|
125
|
-
end
|
|
127
|
+
render_create_failure
|
|
126
128
|
end
|
|
127
|
-
|
|
129
|
+
rescue Bulkrax::UnsafeCloudSourceError, Bulkrax::UnsafePathError => e
|
|
130
|
+
reject_unsafe_cloud_files(e)
|
|
128
131
|
end
|
|
129
132
|
# rubocop:enable Metrics/AbcSize
|
|
130
133
|
|
|
@@ -142,11 +145,19 @@ module Bulkrax
|
|
|
142
145
|
# Skipped during a continue
|
|
143
146
|
field_mapping_params if params[:importer][:parser_fields].present?
|
|
144
147
|
|
|
145
|
-
|
|
146
|
-
|
|
148
|
+
# Assign the proposed attributes in memory first, so an unsafe cloud
|
|
149
|
+
# selection is rejected before anything is persisted; the rescue
|
|
150
|
+
# below then leaves the importer's persisted attributes untouched.
|
|
151
|
+
@importer.assign_attributes(importer_params)
|
|
152
|
+
preflight_cloud_files!(cloud_files)
|
|
153
|
+
|
|
154
|
+
if @importer.save
|
|
155
|
+
return render_update_failure if files_for_import(file, cloud_files, uploads) && !@importer.save
|
|
156
|
+
return render_update_failure if params[:commit] != 'Update Importer' && !set_files_parser_fields
|
|
157
|
+
dispatch_cloud_downloads!
|
|
158
|
+
|
|
147
159
|
# do not perform the import
|
|
148
160
|
unless params[:commit] == 'Update Importer'
|
|
149
|
-
set_files_parser_fields
|
|
150
161
|
Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id, update_harvest)
|
|
151
162
|
end
|
|
152
163
|
if api_request?
|
|
@@ -155,12 +166,10 @@ module Bulkrax
|
|
|
155
166
|
redirect_to importers_path, notice: 'Importer was successfully updated.'
|
|
156
167
|
end
|
|
157
168
|
else
|
|
158
|
-
|
|
159
|
-
json_response('update', :unprocessable_entity, 'Something went wrong.')
|
|
160
|
-
else
|
|
161
|
-
render :edit
|
|
162
|
-
end
|
|
169
|
+
render_update_failure
|
|
163
170
|
end
|
|
171
|
+
rescue Bulkrax::UnsafeCloudSourceError, Bulkrax::UnsafePathError => e
|
|
172
|
+
reject_unsafe_cloud_files(e)
|
|
164
173
|
end
|
|
165
174
|
# rubocop:enable all
|
|
166
175
|
# rubocop:enable Metrics/MethodLength
|
|
@@ -332,6 +341,36 @@ module Bulkrax
|
|
|
332
341
|
'text/csv'
|
|
333
342
|
end
|
|
334
343
|
|
|
344
|
+
# Reports an unsafe cloud-file batch without leaking the error's class
|
|
345
|
+
# or backtrace (which could include a provider auth header) to the
|
|
346
|
+
# response; `e.message` is always one of our own static, credential-free
|
|
347
|
+
# strings (see Bulkrax::CloudSourceValidator / Bulkrax::SafePath).
|
|
348
|
+
def reject_unsafe_cloud_files(e)
|
|
349
|
+
Rails.logger.error("Bulkrax: rejected unsafe cloud file batch for importer #{@importer&.id}: #{e.class}: #{e.message}")
|
|
350
|
+
if api_request?
|
|
351
|
+
json_response('create', :unprocessable_entity, e.message)
|
|
352
|
+
else
|
|
353
|
+
flash[:error] = e.message
|
|
354
|
+
redirect_back fallback_location: bulkrax.importers_path
|
|
355
|
+
end
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
def render_create_failure
|
|
359
|
+
if api_request?
|
|
360
|
+
json_response('create', :unprocessable_entity)
|
|
361
|
+
else
|
|
362
|
+
render :new
|
|
363
|
+
end
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
def render_update_failure
|
|
367
|
+
if api_request?
|
|
368
|
+
json_response('update', :unprocessable_entity, 'Something went wrong.')
|
|
369
|
+
else
|
|
370
|
+
render :edit
|
|
371
|
+
end
|
|
372
|
+
end
|
|
373
|
+
|
|
335
374
|
def render_request(message, validate_only = false)
|
|
336
375
|
if api_request?
|
|
337
376
|
json_response('create', :created, message)
|
|
@@ -347,6 +386,8 @@ module Bulkrax
|
|
|
347
386
|
params[:commit] == 'Update and Harvest Updated Items'
|
|
348
387
|
end
|
|
349
388
|
|
|
389
|
+
# @return [Boolean] the result of the save, so the caller can treat a
|
|
390
|
+
# failure here like an ordinary validation failure
|
|
350
391
|
def set_files_parser_fields
|
|
351
392
|
@importer.parser_fields['update_files'] =
|
|
352
393
|
@importer.parser_fields['replace_files'] =
|
|
@@ -359,7 +400,7 @@ module Bulkrax
|
|
|
359
400
|
elsif params[:commit] == 'Remove and Rerun'
|
|
360
401
|
@importer.parser_fields['remove_and_rerun'] = true
|
|
361
402
|
elsif params[:commit] == 'Update and Harvest Updated Items'
|
|
362
|
-
return
|
|
403
|
+
return true
|
|
363
404
|
else
|
|
364
405
|
@importer.parser_fields['metadata_only'] = true
|
|
365
406
|
end
|
|
@@ -139,21 +139,38 @@ module Bulkrax
|
|
|
139
139
|
csv_by_depth[shallowest_depth]
|
|
140
140
|
end
|
|
141
141
|
|
|
142
|
-
#
|
|
142
|
+
# Validates every selected cloud-file entry — raw name and source
|
|
143
|
+
# safety — before the importer is created or updated at all. Must not
|
|
144
|
+
# touch the importer's workspace (no path_for_import, no directory
|
|
145
|
+
# creation, no download): for #create that workspace doesn't exist
|
|
146
|
+
# yet, and for #update it must not be assumed safe to touch before the
|
|
147
|
+
# proposed attributes are known to be acceptable.
|
|
148
|
+
#
|
|
149
|
+
# @param cloud_files [Hash, nil] cloud file specs from browse-everything
|
|
150
|
+
# @raise [Bulkrax::UnsafePathError, Bulkrax::UnsafeCloudSourceError] on the first unsafe entry
|
|
151
|
+
def preflight_cloud_files!(cloud_files)
|
|
152
|
+
return if cloud_files.blank?
|
|
153
|
+
|
|
154
|
+
cloud_files.values.each { |file| Bulkrax::CloudSourceValidator.preflight!(file) }
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Assigns files/cloud selections onto the importer record's in-memory
|
|
158
|
+
# parser fields, but does not save: the caller (ImportersController)
|
|
159
|
+
# owns the second save, so it can check its result and avoid rendering
|
|
160
|
+
# success or enqueueing an import on failure.
|
|
161
|
+
#
|
|
143
162
|
# @param file [ActionDispatch::Http::UploadedFile, nil] a directly uploaded file
|
|
144
163
|
# @param cloud_files [Hash, nil] cloud file paths from browse-everything
|
|
145
164
|
# @param uploads [ActiveRecord::Relation, Array, nil] Hyrax::UploadedFile records
|
|
165
|
+
# @return [Boolean] true if any parser field was touched and a second save is needed
|
|
146
166
|
def files_for_import(file, cloud_files, uploads)
|
|
147
|
-
|
|
167
|
+
@cloud_download_plan = nil
|
|
168
|
+
return false if file.blank? && cloud_files.blank? && uploads.blank?
|
|
148
169
|
|
|
149
170
|
@importer[:parser_fields]['import_file_path'] = @importer.parser.write_import_file(file) if file.present?
|
|
150
171
|
if cloud_files.present?
|
|
151
172
|
@importer[:parser_fields]['cloud_file_paths'] = cloud_files
|
|
152
|
-
|
|
153
|
-
# For CSV, we expect only file uploads, so we won't get the file_path back
|
|
154
|
-
# and we expect the import_file_path to be set already
|
|
155
|
-
target = @importer.parser.retrieve_cloud_files(cloud_files, @importer)
|
|
156
|
-
@importer[:parser_fields]['import_file_path'] = target if target.present?
|
|
173
|
+
assign_cloud_file_paths(cloud_files)
|
|
157
174
|
end
|
|
158
175
|
|
|
159
176
|
if uploads.present?
|
|
@@ -162,7 +179,36 @@ module Bulkrax
|
|
|
162
179
|
end
|
|
163
180
|
end
|
|
164
181
|
|
|
165
|
-
|
|
182
|
+
true
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# For a parser that separates planning from dispatch (currently only
|
|
186
|
+
# CsvParser), resolves and validates the whole batch and records the
|
|
187
|
+
# target paths, deferring enqueue until {#dispatch_cloud_downloads!} is
|
|
188
|
+
# called after the controller persists those paths. Other parsers
|
|
189
|
+
# (e.g. BagitParser) already complete cloud retrieval synchronously
|
|
190
|
+
# and return the resolved path directly, so there's nothing left to
|
|
191
|
+
# dispatch.
|
|
192
|
+
def assign_cloud_file_paths(cloud_files)
|
|
193
|
+
parser = @importer.parser
|
|
194
|
+
if parser.respond_to?(:plan_cloud_files)
|
|
195
|
+
@cloud_download_plan = parser.plan_cloud_files(cloud_files)
|
|
196
|
+
@importer[:parser_fields]['original_file_paths'] =
|
|
197
|
+
@cloud_download_plan.map { |_file, _relative_filename, target_file| target_file }
|
|
198
|
+
else
|
|
199
|
+
target = parser.retrieve_cloud_files(cloud_files, @importer)
|
|
200
|
+
@importer[:parser_fields]['import_file_path'] = target if target.present?
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# Dispatches a plan prepared by {#assign_cloud_file_paths}, if any.
|
|
205
|
+
# Called by the controller only after it has successfully persisted
|
|
206
|
+
# the resolved paths, so a download is never enqueued for a path the
|
|
207
|
+
# importer doesn't actually have on record.
|
|
208
|
+
def dispatch_cloud_downloads!
|
|
209
|
+
return if @cloud_download_plan.blank?
|
|
210
|
+
|
|
211
|
+
@importer.parser.dispatch_cloud_files!(@cloud_download_plan, @importer)
|
|
166
212
|
end
|
|
167
213
|
|
|
168
214
|
def write_files(files)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
# Raised by {Bulkrax::CloudSourceValidator} when a cloud-file source
|
|
5
|
+
# (the `url` a user selected via browse-everything, or supplied directly
|
|
6
|
+
# in importer params) can't be safely downloaded: a scheme other than
|
|
7
|
+
# `file`/`http`/`https`, a `file:` source outside the configured
|
|
8
|
+
# filesystem-provider root, or an HTTP(S) source that resolves to a
|
|
9
|
+
# private, loopback, or link-local/metadata address.
|
|
10
|
+
class UnsafeCloudSourceError < StandardError; end
|
|
11
|
+
end
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
# Raised by {Bulkrax::SafePath} when a path would escape its configured
|
|
5
|
+
# root, whether that root is a ZIP extraction directory, an importer's
|
|
6
|
+
# trusted workspace, or a configured filesystem-provider home.
|
|
7
|
+
class UnsafePathError < StandardError; end
|
|
8
|
+
end
|
|
@@ -1,31 +1,166 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
+
|
|
2
3
|
module Bulkrax
|
|
3
4
|
class DownloadCloudFileJob < ApplicationJob
|
|
4
5
|
queue_as Bulkrax.config.ingest_queue_name
|
|
5
6
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
#
|
|
9
|
-
#
|
|
7
|
+
# Retrieve a cloud file and write it to the importer's workspace.
|
|
8
|
+
#
|
|
9
|
+
# `importer_id:` and `relative_filename:` are required keywords: the
|
|
10
|
+
# destination is always recomputed here, confined to this importer's
|
|
11
|
+
# own workspace, rather than trusted from a serialized job argument.
|
|
12
|
+
# `target_file` is a deprecated positional placeholder, kept only so
|
|
13
|
+
# that a pre-upgrade call shape (`file`, `target_file`) still binds
|
|
14
|
+
# without Ruby raising its own ArgumentError — it is never read, even
|
|
15
|
+
# when supplied alongside otherwise-valid keywords, so it can't be
|
|
16
|
+
# used to redirect the destination. A call made this way is missing
|
|
17
|
+
# the required keywords, so it raises below rather than running.
|
|
18
|
+
#
|
|
19
|
+
# Note: if using the file system provider, the mounted directory in
|
|
10
20
|
# browse_everything MUST be shared by web and worker servers
|
|
11
|
-
def perform(file, target_file)
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
21
|
+
def perform(file, target_file = nil, importer_id: nil, relative_filename: nil)
|
|
22
|
+
raise Bulkrax::UnsafePathError, I18n.t('bulkrax.importer.browse_everything.errors.missing_destination_context') if importer_id.blank? || relative_filename.blank?
|
|
23
|
+
|
|
24
|
+
# `file` and `relative_filename` are independent arguments and may
|
|
25
|
+
# not agree (a tampered or replayed job call could supply either
|
|
26
|
+
# one separately) — check the raw file_name directly rather than
|
|
27
|
+
# assuming it was already validated by whatever enqueued this job.
|
|
28
|
+
Bulkrax::SafePath.reject_unsafe_entry!(file['file_name'])
|
|
29
|
+
|
|
30
|
+
importer = Bulkrax::Importer.find(importer_id)
|
|
31
|
+
target_file = Bulkrax::SafePath.confine!(importer.parser.path_for_import, relative_filename, require_file: true)
|
|
32
|
+
|
|
33
|
+
# Re-validate the source here too: this job may run on a different
|
|
34
|
+
# process/host than the one that enqueued it, and could be invoked
|
|
35
|
+
# directly with a tampered or stale `file` argument.
|
|
36
|
+
Bulkrax::CloudSourceValidator.validate!(file)
|
|
37
|
+
|
|
38
|
+
download(file, target_file)
|
|
39
|
+
Rails.logger.info "Bulkrax: cloud file download complete for #{file['file_name']}"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
# Downloads through a temp file in the *same* directory as
|
|
45
|
+
# `target_file` (so the final move is same-filesystem and atomic),
|
|
46
|
+
# named with a random suffix rather than anything derived from
|
|
47
|
+
# caller-controlled input, and removed on any failure. This means a
|
|
48
|
+
# failed or non-success download can never truncate or corrupt an
|
|
49
|
+
# already-good destination file from a previous successful import.
|
|
50
|
+
def download(file, target_file)
|
|
51
|
+
temp_file = "#{target_file}.#{SecureRandom.hex(8)}.part"
|
|
52
|
+
uri = Addressable::URI.parse(file['url'].to_s)
|
|
53
|
+
if uri.scheme == 'file'
|
|
54
|
+
download_file_source(uri, temp_file)
|
|
55
|
+
else
|
|
56
|
+
download_http_source(file, URI.parse(file['url'].to_s), temp_file)
|
|
57
|
+
end
|
|
58
|
+
FileUtils.mv(temp_file, target_file)
|
|
59
|
+
ensure
|
|
60
|
+
FileUtils.rm_f(temp_file) if temp_file && File.exist?(temp_file)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# The source was already confined to the configured filesystem
|
|
64
|
+
# provider's root by {Bulkrax::CloudSourceValidator}.
|
|
65
|
+
def download_file_source(uri, temp_file)
|
|
66
|
+
source_path = URI::DEFAULT_PARSER.unescape(uri.path.to_s)
|
|
67
|
+
IO.copy_stream(source_path, temp_file)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Resolved, IP-pinned, redirect-free, and streamed straight to disk in
|
|
71
|
+
# bounded-size chunks — rather than handing the URL back to a transport
|
|
72
|
+
# (browse-everything's Retriever) that re-resolves the hostname with no
|
|
73
|
+
# SSRF protection at all. This is a *separate* connection from the one
|
|
74
|
+
# {Bulkrax::CloudSourceValidator}'s HEAD check made; each independently
|
|
75
|
+
# applies the same validation at its own connection time, so there is
|
|
76
|
+
# no single "pinned connection" shared between validation and download.
|
|
77
|
+
#
|
|
78
|
+
# ssrf_filter 1.0.8's public `get`/`head` API has no response-streaming
|
|
79
|
+
# hook: `fetch_once` calls `Net::HTTP#request` with no block, which
|
|
80
|
+
# reads the entire response body into memory before returning; the
|
|
81
|
+
# block `get`/`head` accept is for customizing the *outgoing* request,
|
|
82
|
+
# not consuming the response. So we can't use `SsrfFilter.get` here
|
|
83
|
+
# without buffering the whole remote file in memory first.
|
|
84
|
+
#
|
|
85
|
+
# Reuse ssrf_filter's resolver and IP policy, then pin the validated
|
|
86
|
+
# address with Net::HTTP#ipaddr=. Keeping the original hostname on
|
|
87
|
+
# the HTTP connection preserves native TLS SNI and certificate
|
|
88
|
+
# verification without relying on ssrf_filter's TLS monkeypatches.
|
|
89
|
+
def download_http_source(file, uri, temp_file)
|
|
90
|
+
headers = Bulkrax::CloudSourceValidator.request_headers(file)
|
|
91
|
+
stream_download(uri, headers) { |response| handle_response(file, response, temp_file) }
|
|
92
|
+
rescue SsrfFilter::Error
|
|
93
|
+
# The validator's own HEAD check already ran, but DNS can change
|
|
94
|
+
# between that check and this GET — same rationale as
|
|
95
|
+
# Bulkrax::CloudSourceValidator#validate_http_scheme!, and the same
|
|
96
|
+
# static, class-agnostic message for the same reason.
|
|
97
|
+
raise Bulkrax::UnsafeCloudSourceError, 'blocked unsafe remote source'
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def handle_response(file, response, temp_file)
|
|
101
|
+
unless response.is_a?(Net::HTTPSuccess)
|
|
102
|
+
# Static on purpose — this becomes the importer's status message,
|
|
103
|
+
# not just an ops log; see Bulkrax::CloudSourceValidator for why
|
|
104
|
+
# status codes/response details aren't echoed into it. The status
|
|
105
|
+
# code itself is logged (not attacker-controlled, not a credential).
|
|
106
|
+
Rails.logger.error("Bulkrax: cloud file download failed for #{file['file_name']} with status #{response.code}")
|
|
107
|
+
raise Bulkrax::UnsafeCloudSourceError, 'download failed: non-success response'
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
File.open(temp_file, 'wb') { |io| response.read_body { |chunk| io.write(chunk) } }
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Resolves `uri`'s host, rejects it unless it has at least one public
|
|
114
|
+
# IP (reusing ssrf_filter's own resolver/blacklist — see above), and
|
|
115
|
+
# connects directly to that IP: with the environment proxy explicitly
|
|
116
|
+
# disabled (`Net::HTTP.new`'s `p_addr` defaults to `:ENV`, which would
|
|
117
|
+
# otherwise route through `http_proxy`/`https_proxy` and hand the
|
|
118
|
+
# pinned connection to a third party), and with the real hostname sent
|
|
119
|
+
# as the `Host` header and used for TLS SNI/certificate verification
|
|
120
|
+
# instead of the IP. Yields the live `Net::HTTPResponse` to the block
|
|
121
|
+
# *before* its body is read, so the caller can stream it.
|
|
122
|
+
def stream_download(uri, headers)
|
|
123
|
+
raise SsrfFilter::InvalidUriScheme, "URI scheme '#{uri.scheme}' not in whitelist" unless
|
|
124
|
+
SsrfFilter::DEFAULT_SCHEME_WHITELIST.include?(uri.scheme)
|
|
125
|
+
|
|
126
|
+
hostname = uri.hostname
|
|
127
|
+
ip_addresses = SsrfFilter::DEFAULT_RESOLVER.call(hostname)
|
|
128
|
+
raise SsrfFilter::UnresolvedHostname, "Could not resolve hostname '#{hostname}'" if ip_addresses.empty?
|
|
129
|
+
|
|
130
|
+
public_addresses = ip_addresses.reject { |ip| unsafe_ip_address?(ip) }
|
|
131
|
+
raise SsrfFilter::PrivateIPAddress, "Hostname '#{hostname}' has no public ip addresses" if public_addresses.empty?
|
|
132
|
+
|
|
133
|
+
request = Net::HTTP::Get.new(uri)
|
|
134
|
+
request['host'] = host_header(hostname, uri)
|
|
135
|
+
headers.each { |key, value| request[key] = value }
|
|
136
|
+
|
|
137
|
+
http = Net::HTTP.new(hostname, uri.port, nil)
|
|
138
|
+
http.ipaddr = public_addresses.sample.to_s
|
|
139
|
+
http.use_ssl = (uri.scheme == 'https')
|
|
140
|
+
http.start { |connection| connection.request(request) { |response| yield response } }
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Mirrors `SsrfFilter.unsafe_ip_address?`, but against its public
|
|
144
|
+
# `IPV4_BLACKLIST`/`IPV6_BLACKLIST` constants directly rather than
|
|
145
|
+
# calling that private method — we only ever check single resolved
|
|
146
|
+
# addresses here (never a CIDR range), so the masked-range guard it
|
|
147
|
+
# also carries doesn't apply.
|
|
148
|
+
def unsafe_ip_address?(ip_address)
|
|
149
|
+
return SsrfFilter::IPV4_BLACKLIST.any? { |range| range.include?(ip_address) } if ip_address.ipv4?
|
|
150
|
+
return SsrfFilter::IPV6_BLACKLIST.any? { |range| range.include?(ip_address) } if ip_address.ipv6?
|
|
151
|
+
|
|
152
|
+
true
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# Matches `SsrfFilter.host_header`: the `Host` header should carry the
|
|
156
|
+
# real hostname (not the pinned IP we actually connect to), with the
|
|
157
|
+
# port suffixed only when it isn't the scheme's default.
|
|
158
|
+
def host_header(hostname, uri)
|
|
159
|
+
if (uri.port == 80 && uri.scheme == 'http') || (uri.port == 443 && uri.scheme == 'https')
|
|
160
|
+
hostname
|
|
161
|
+
else
|
|
162
|
+
"#{hostname}:#{uri.port}"
|
|
27
163
|
end
|
|
28
|
-
Rails.logger.info "Download complete: #{file['url']} to #{target_file}"
|
|
29
164
|
end
|
|
30
165
|
end
|
|
31
166
|
end
|
|
@@ -472,27 +472,27 @@ module Bulkrax
|
|
|
472
472
|
# Zip Slip preflight — reject entries whose names are obviously unsafe
|
|
473
473
|
# (absolute paths, `..` segments) before we touch the filesystem.
|
|
474
474
|
# {#safe_extract_path} is the final line of defense; this check just
|
|
475
|
-
# fails fast with a clear message.
|
|
475
|
+
# fails fast with a clear message. Delegates to {Bulkrax::SafePath} so
|
|
476
|
+
# cloud-file destinations and sources share the same rules.
|
|
476
477
|
#
|
|
477
478
|
# @raise [Bulkrax::UnzipError] if the entry name is unsafe
|
|
478
479
|
def reject_unsafe_entry!(name)
|
|
479
|
-
|
|
480
|
-
|
|
480
|
+
Bulkrax::SafePath.reject_unsafe_entry!(name)
|
|
481
|
+
rescue Bulkrax::UnsafePathError
|
|
482
|
+
raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: name.dup.force_encoding(Encoding::UTF_8).scrub)
|
|
481
483
|
end
|
|
482
484
|
|
|
483
485
|
# Zip Slip chokepoint. Resolves `relative_dest` against `dest_dir` and
|
|
484
486
|
# returns the absolute destination path — but only if it stays inside
|
|
485
|
-
# `dest_dir
|
|
486
|
-
# own path with `File.join`, so the
|
|
487
|
-
# construction.
|
|
487
|
+
# `dest_dir`, including through any symlink. Callers must use this
|
|
488
|
+
# value rather than building their own path with `File.join`, so the
|
|
489
|
+
# path returned is always safe by construction.
|
|
488
490
|
#
|
|
489
491
|
# @return [String] absolute destination path, validated to be inside `dest_dir`
|
|
490
492
|
# @raise [Bulkrax::UnzipError] if `relative_dest` escapes `dest_dir`
|
|
491
493
|
def safe_extract_path(dest_dir, relative_dest)
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
return dest_path if dest_path == expanded_dest_dir
|
|
495
|
-
return dest_path if dest_path.start_with?("#{expanded_dest_dir}#{File::SEPARATOR}")
|
|
494
|
+
Bulkrax::SafePath.confine!(dest_dir, relative_dest)
|
|
495
|
+
rescue Bulkrax::UnsafePathError
|
|
496
496
|
raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: relative_dest)
|
|
497
497
|
end
|
|
498
498
|
|