bulkrax 10.0.0 → 10.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/controllers/bulkrax/importers_controller.rb +58 -17
- data/app/controllers/concerns/bulkrax/importer_file_handler.rb +54 -8
- data/app/errors/bulkrax/unsafe_cloud_source_error.rb +11 -0
- data/app/errors/bulkrax/unsafe_path_error.rb +8 -0
- data/app/jobs/bulkrax/download_cloud_file_job.rb +156 -21
- data/app/parsers/bulkrax/application_parser.rb +11 -11
- data/app/parsers/bulkrax/application_parser.rb.orig +545 -0
- data/app/parsers/bulkrax/bagit_parser.rb +10 -3
- data/app/parsers/bulkrax/csv_parser.rb +46 -15
- data/app/services/bulkrax/cloud_source_validator.rb +123 -0
- data/app/services/bulkrax/safe_path.rb +129 -0
- data/config/locales/bulkrax.en.yml +2 -0
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +1 -0
- metadata +27 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7c070c4c3642a68c18649c07fdf78927263383409488dbef7a29853c77127549
|
|
4
|
+
data.tar.gz: 4c29abd0313b29256af87bf8f38550092120dd577933b3b0095d3d0046c64c21
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b693f17e5e5e3e47a6f858300a4d4cdeba454e8e6a77746b82cf76973c38920b96bc1658cf979cc77551d7f1843400560ad35b002952911902384d3901c23970
|
|
7
|
+
data.tar.gz: 3ae0cc2204b40914a9fd027ce65acc0ee9d1d4b30bf04f9ebe832cb362d60a139f2ec2a9f2d1cc10051c1d5f89897895060ab1eeb4571404a9f4663e8b4d4397
|
|
@@ -101,7 +101,6 @@ module Bulkrax
|
|
|
101
101
|
# rubocop:disable Metrics/MethodLength
|
|
102
102
|
# rubocop:disable Metrics/AbcSize
|
|
103
103
|
def create
|
|
104
|
-
# rubocop:disable Style/IfInsideElse
|
|
105
104
|
if api_request?
|
|
106
105
|
return return_json_response unless valid_create_params?
|
|
107
106
|
# authenticate_and_authorize! doesn't build for API requests; build it here
|
|
@@ -117,8 +116,15 @@ module Bulkrax
|
|
|
117
116
|
# the following line is needed to handle updating remote files of a FileSet
|
|
118
117
|
# on a new import otherwise it only gets updated during the update path
|
|
119
118
|
@importer.parser_fields['update_files'] = true if params[:commit] == 'Create and Import'
|
|
119
|
+
|
|
120
|
+
# Reject an unsafe cloud selection before the importer is ever
|
|
121
|
+
# persisted, rather than after save when a row would already exist.
|
|
122
|
+
preflight_cloud_files!(cloud_files)
|
|
123
|
+
|
|
120
124
|
if @importer.save
|
|
121
|
-
files_for_import(file, cloud_files, uploads)
|
|
125
|
+
return render_create_failure if files_for_import(file, cloud_files, uploads) && !@importer.save
|
|
126
|
+
dispatch_cloud_downloads!
|
|
127
|
+
|
|
122
128
|
if params[:commit] == 'Create and Import'
|
|
123
129
|
Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id)
|
|
124
130
|
render_request('Importer was successfully created and import has been queued.')
|
|
@@ -129,13 +135,10 @@ module Bulkrax
|
|
|
129
135
|
render_request('Importer was successfully created.')
|
|
130
136
|
end
|
|
131
137
|
else
|
|
132
|
-
|
|
133
|
-
json_response('create', :unprocessable_entity)
|
|
134
|
-
else
|
|
135
|
-
render :new
|
|
136
|
-
end
|
|
138
|
+
render_create_failure
|
|
137
139
|
end
|
|
138
|
-
|
|
140
|
+
rescue Bulkrax::UnsafeCloudSourceError, Bulkrax::UnsafePathError => e
|
|
141
|
+
reject_unsafe_cloud_files(e)
|
|
139
142
|
end
|
|
140
143
|
# rubocop:enable Metrics/AbcSize
|
|
141
144
|
|
|
@@ -153,11 +156,19 @@ module Bulkrax
|
|
|
153
156
|
# Skipped during a continue
|
|
154
157
|
field_mapping_params if params[:importer][:parser_fields].present?
|
|
155
158
|
|
|
156
|
-
|
|
157
|
-
|
|
159
|
+
# Assign the proposed attributes in memory first, so an unsafe cloud
|
|
160
|
+
# selection is rejected before anything is persisted; the rescue
|
|
161
|
+
# below then leaves the importer's persisted attributes untouched.
|
|
162
|
+
@importer.assign_attributes(importer_params)
|
|
163
|
+
preflight_cloud_files!(cloud_files)
|
|
164
|
+
|
|
165
|
+
if @importer.save
|
|
166
|
+
return render_update_failure if files_for_import(file, cloud_files, uploads) && !@importer.save
|
|
167
|
+
return render_update_failure if params[:commit] != 'Update Importer' && !set_files_parser_fields
|
|
168
|
+
dispatch_cloud_downloads!
|
|
169
|
+
|
|
158
170
|
# do not perform the import
|
|
159
171
|
unless params[:commit] == 'Update Importer'
|
|
160
|
-
set_files_parser_fields
|
|
161
172
|
Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id, update_harvest)
|
|
162
173
|
end
|
|
163
174
|
if api_request?
|
|
@@ -166,12 +177,10 @@ module Bulkrax
|
|
|
166
177
|
redirect_to importers_path, notice: 'Importer was successfully updated.'
|
|
167
178
|
end
|
|
168
179
|
else
|
|
169
|
-
|
|
170
|
-
json_response('update', :unprocessable_entity, 'Something went wrong.')
|
|
171
|
-
else
|
|
172
|
-
render :edit
|
|
173
|
-
end
|
|
180
|
+
render_update_failure
|
|
174
181
|
end
|
|
182
|
+
rescue Bulkrax::UnsafeCloudSourceError, Bulkrax::UnsafePathError => e
|
|
183
|
+
reject_unsafe_cloud_files(e)
|
|
175
184
|
end
|
|
176
185
|
# rubocop:enable all
|
|
177
186
|
# rubocop:enable Metrics/MethodLength
|
|
@@ -333,6 +342,36 @@ module Bulkrax
|
|
|
333
342
|
'text/csv'
|
|
334
343
|
end
|
|
335
344
|
|
|
345
|
+
# Reports an unsafe cloud-file batch without leaking the error's class
|
|
346
|
+
# or backtrace (which could include a provider auth header) to the
|
|
347
|
+
# response; `e.message` is always one of our own static, credential-free
|
|
348
|
+
# strings (see Bulkrax::CloudSourceValidator / Bulkrax::SafePath).
|
|
349
|
+
def reject_unsafe_cloud_files(e)
|
|
350
|
+
Rails.logger.error("Bulkrax: rejected unsafe cloud file batch for importer #{@importer&.id}: #{e.class}: #{e.message}")
|
|
351
|
+
if api_request?
|
|
352
|
+
json_response('create', :unprocessable_entity, e.message)
|
|
353
|
+
else
|
|
354
|
+
flash[:error] = e.message
|
|
355
|
+
redirect_back fallback_location: bulkrax.importers_path
|
|
356
|
+
end
|
|
357
|
+
end
|
|
358
|
+
|
|
359
|
+
def render_create_failure
|
|
360
|
+
if api_request?
|
|
361
|
+
json_response('create', :unprocessable_entity)
|
|
362
|
+
else
|
|
363
|
+
render :new
|
|
364
|
+
end
|
|
365
|
+
end
|
|
366
|
+
|
|
367
|
+
def render_update_failure
|
|
368
|
+
if api_request?
|
|
369
|
+
json_response('update', :unprocessable_entity, 'Something went wrong.')
|
|
370
|
+
else
|
|
371
|
+
render :edit
|
|
372
|
+
end
|
|
373
|
+
end
|
|
374
|
+
|
|
336
375
|
def render_request(message, validate_only = false)
|
|
337
376
|
if api_request?
|
|
338
377
|
json_response('create', :created, message)
|
|
@@ -348,6 +387,8 @@ module Bulkrax
|
|
|
348
387
|
params[:commit] == 'Update and Harvest Updated Items'
|
|
349
388
|
end
|
|
350
389
|
|
|
390
|
+
# @return [Boolean] the result of the save, so the caller can treat a
|
|
391
|
+
# failure here like an ordinary validation failure
|
|
351
392
|
def set_files_parser_fields
|
|
352
393
|
@importer.parser_fields['update_files'] =
|
|
353
394
|
@importer.parser_fields['replace_files'] =
|
|
@@ -360,7 +401,7 @@ module Bulkrax
|
|
|
360
401
|
elsif params[:commit] == 'Remove and Rerun'
|
|
361
402
|
@importer.parser_fields['remove_and_rerun'] = true
|
|
362
403
|
elsif params[:commit] == 'Update and Harvest Updated Items'
|
|
363
|
-
return
|
|
404
|
+
return true
|
|
364
405
|
else
|
|
365
406
|
@importer.parser_fields['metadata_only'] = true
|
|
366
407
|
end
|
|
@@ -139,21 +139,38 @@ module Bulkrax
|
|
|
139
139
|
csv_by_depth[shallowest_depth]
|
|
140
140
|
end
|
|
141
141
|
|
|
142
|
-
#
|
|
142
|
+
# Validates every selected cloud-file entry — raw name and source
|
|
143
|
+
# safety — before the importer is created or updated at all. Must not
|
|
144
|
+
# touch the importer's workspace (no path_for_import, no directory
|
|
145
|
+
# creation, no download): for #create that workspace doesn't exist
|
|
146
|
+
# yet, and for #update it must not be assumed safe to touch before the
|
|
147
|
+
# proposed attributes are known to be acceptable.
|
|
148
|
+
#
|
|
149
|
+
# @param cloud_files [Hash, nil] cloud file specs from browse-everything
|
|
150
|
+
# @raise [Bulkrax::UnsafePathError, Bulkrax::UnsafeCloudSourceError] on the first unsafe entry
|
|
151
|
+
def preflight_cloud_files!(cloud_files)
|
|
152
|
+
return if cloud_files.blank?
|
|
153
|
+
|
|
154
|
+
cloud_files.values.each { |file| Bulkrax::CloudSourceValidator.preflight!(file) }
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Assigns files/cloud selections onto the importer record's in-memory
|
|
158
|
+
# parser fields, but does not save: the caller (ImportersController)
|
|
159
|
+
# owns the second save, so it can check its result and avoid rendering
|
|
160
|
+
# success or enqueueing an import on failure.
|
|
161
|
+
#
|
|
143
162
|
# @param file [ActionDispatch::Http::UploadedFile, nil] a directly uploaded file
|
|
144
163
|
# @param cloud_files [Hash, nil] cloud file paths from browse-everything
|
|
145
164
|
# @param uploads [ActiveRecord::Relation, Array, nil] Hyrax::UploadedFile records
|
|
165
|
+
# @return [Boolean] true if any parser field was touched and a second save is needed
|
|
146
166
|
def files_for_import(file, cloud_files, uploads)
|
|
147
|
-
|
|
167
|
+
@cloud_download_plan = nil
|
|
168
|
+
return false if file.blank? && cloud_files.blank? && uploads.blank?
|
|
148
169
|
|
|
149
170
|
@importer[:parser_fields]['import_file_path'] = @importer.parser.write_import_file(file) if file.present?
|
|
150
171
|
if cloud_files.present?
|
|
151
172
|
@importer[:parser_fields]['cloud_file_paths'] = cloud_files
|
|
152
|
-
|
|
153
|
-
# For CSV, we expect only file uploads, so we won't get the file_path back
|
|
154
|
-
# and we expect the import_file_path to be set already
|
|
155
|
-
target = @importer.parser.retrieve_cloud_files(cloud_files, @importer)
|
|
156
|
-
@importer[:parser_fields]['import_file_path'] = target if target.present?
|
|
173
|
+
assign_cloud_file_paths(cloud_files)
|
|
157
174
|
end
|
|
158
175
|
|
|
159
176
|
if uploads.present?
|
|
@@ -162,7 +179,36 @@ module Bulkrax
|
|
|
162
179
|
end
|
|
163
180
|
end
|
|
164
181
|
|
|
165
|
-
|
|
182
|
+
true
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# For a parser that separates planning from dispatch (currently only
|
|
186
|
+
# CsvParser), resolves and validates the whole batch and records the
|
|
187
|
+
# target paths, deferring enqueue until {#dispatch_cloud_downloads!} is
|
|
188
|
+
# called after the controller persists those paths. Other parsers
|
|
189
|
+
# (e.g. BagitParser) already complete cloud retrieval synchronously
|
|
190
|
+
# and return the resolved path directly, so there's nothing left to
|
|
191
|
+
# dispatch.
|
|
192
|
+
def assign_cloud_file_paths(cloud_files)
|
|
193
|
+
parser = @importer.parser
|
|
194
|
+
if parser.respond_to?(:plan_cloud_files)
|
|
195
|
+
@cloud_download_plan = parser.plan_cloud_files(cloud_files)
|
|
196
|
+
@importer[:parser_fields]['original_file_paths'] =
|
|
197
|
+
@cloud_download_plan.map { |_file, _relative_filename, target_file| target_file }
|
|
198
|
+
else
|
|
199
|
+
target = parser.retrieve_cloud_files(cloud_files, @importer)
|
|
200
|
+
@importer[:parser_fields]['import_file_path'] = target if target.present?
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# Dispatches a plan prepared by {#assign_cloud_file_paths}, if any.
|
|
205
|
+
# Called by the controller only after it has successfully persisted
|
|
206
|
+
# the resolved paths, so a download is never enqueued for a path the
|
|
207
|
+
# importer doesn't actually have on record.
|
|
208
|
+
def dispatch_cloud_downloads!
|
|
209
|
+
return if @cloud_download_plan.blank?
|
|
210
|
+
|
|
211
|
+
@importer.parser.dispatch_cloud_files!(@cloud_download_plan, @importer)
|
|
166
212
|
end
|
|
167
213
|
|
|
168
214
|
def write_files(files)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
# Raised by {Bulkrax::CloudSourceValidator} when a cloud-file source
|
|
5
|
+
# (the `url` a user selected via browse-everything, or supplied directly
|
|
6
|
+
# in importer params) can't be safely downloaded: a scheme other than
|
|
7
|
+
# `file`/`http`/`https`, a `file:` source outside the configured
|
|
8
|
+
# filesystem-provider root, or an HTTP(S) source that resolves to a
|
|
9
|
+
# private, loopback, or link-local/metadata address.
|
|
10
|
+
class UnsafeCloudSourceError < StandardError; end
|
|
11
|
+
end
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
# Raised by {Bulkrax::SafePath} when a path would escape its configured
|
|
5
|
+
# root, whether that root is a ZIP extraction directory, an importer's
|
|
6
|
+
# trusted workspace, or a configured filesystem-provider home.
|
|
7
|
+
class UnsafePathError < StandardError; end
|
|
8
|
+
end
|
|
@@ -1,31 +1,166 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
+
|
|
2
3
|
module Bulkrax
|
|
3
4
|
class DownloadCloudFileJob < ApplicationJob
|
|
4
5
|
queue_as Bulkrax.config.ingest_queue_name
|
|
5
6
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
#
|
|
9
|
-
#
|
|
7
|
+
# Retrieve a cloud file and write it to the importer's workspace.
|
|
8
|
+
#
|
|
9
|
+
# `importer_id:` and `relative_filename:` are required keywords: the
|
|
10
|
+
# destination is always recomputed here, confined to this importer's
|
|
11
|
+
# own workspace, rather than trusted from a serialized job argument.
|
|
12
|
+
# `target_file` is a deprecated positional placeholder, kept only so
|
|
13
|
+
# that a pre-upgrade call shape (`file`, `target_file`) still binds
|
|
14
|
+
# without Ruby raising its own ArgumentError — it is never read, even
|
|
15
|
+
# when supplied alongside otherwise-valid keywords, so it can't be
|
|
16
|
+
# used to redirect the destination. A call made this way is missing
|
|
17
|
+
# the required keywords, so it raises below rather than running.
|
|
18
|
+
#
|
|
19
|
+
# Note: if using the file system provider, the mounted directory in
|
|
10
20
|
# browse_everything MUST be shared by web and worker servers
|
|
11
|
-
def perform(file, target_file)
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
21
|
+
def perform(file, target_file = nil, importer_id: nil, relative_filename: nil)
|
|
22
|
+
raise Bulkrax::UnsafePathError, I18n.t('bulkrax.importer.browse_everything.errors.missing_destination_context') if importer_id.blank? || relative_filename.blank?
|
|
23
|
+
|
|
24
|
+
# `file` and `relative_filename` are independent arguments and may
|
|
25
|
+
# not agree (a tampered or replayed job call could supply either
|
|
26
|
+
# one separately) — check the raw file_name directly rather than
|
|
27
|
+
# assuming it was already validated by whatever enqueued this job.
|
|
28
|
+
Bulkrax::SafePath.reject_unsafe_entry!(file['file_name'])
|
|
29
|
+
|
|
30
|
+
importer = Bulkrax::Importer.find(importer_id)
|
|
31
|
+
target_file = Bulkrax::SafePath.confine!(importer.parser.path_for_import, relative_filename, require_file: true)
|
|
32
|
+
|
|
33
|
+
# Re-validate the source here too: this job may run on a different
|
|
34
|
+
# process/host than the one that enqueued it, and could be invoked
|
|
35
|
+
# directly with a tampered or stale `file` argument.
|
|
36
|
+
Bulkrax::CloudSourceValidator.validate!(file)
|
|
37
|
+
|
|
38
|
+
download(file, target_file)
|
|
39
|
+
Rails.logger.info "Bulkrax: cloud file download complete for #{file['file_name']}"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
# Downloads through a temp file in the *same* directory as
|
|
45
|
+
# `target_file` (so the final move is same-filesystem and atomic),
|
|
46
|
+
# named with a random suffix rather than anything derived from
|
|
47
|
+
# caller-controlled input, and removed on any failure. This means a
|
|
48
|
+
# failed or non-success download can never truncate or corrupt an
|
|
49
|
+
# already-good destination file from a previous successful import.
|
|
50
|
+
def download(file, target_file)
|
|
51
|
+
temp_file = "#{target_file}.#{SecureRandom.hex(8)}.part"
|
|
52
|
+
uri = Addressable::URI.parse(file['url'].to_s)
|
|
53
|
+
if uri.scheme == 'file'
|
|
54
|
+
download_file_source(uri, temp_file)
|
|
55
|
+
else
|
|
56
|
+
download_http_source(file, URI.parse(file['url'].to_s), temp_file)
|
|
57
|
+
end
|
|
58
|
+
FileUtils.mv(temp_file, target_file)
|
|
59
|
+
ensure
|
|
60
|
+
FileUtils.rm_f(temp_file) if temp_file && File.exist?(temp_file)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# The source was already confined to the configured filesystem
|
|
64
|
+
# provider's root by {Bulkrax::CloudSourceValidator}.
|
|
65
|
+
def download_file_source(uri, temp_file)
|
|
66
|
+
source_path = URI::DEFAULT_PARSER.unescape(uri.path.to_s)
|
|
67
|
+
IO.copy_stream(source_path, temp_file)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Resolved, IP-pinned, redirect-free, and streamed straight to disk in
|
|
71
|
+
# bounded-size chunks — rather than handing the URL back to a transport
|
|
72
|
+
# (browse-everything's Retriever) that re-resolves the hostname with no
|
|
73
|
+
# SSRF protection at all. This is a *separate* connection from the one
|
|
74
|
+
# {Bulkrax::CloudSourceValidator}'s HEAD check made; each independently
|
|
75
|
+
# applies the same validation at its own connection time, so there is
|
|
76
|
+
# no single "pinned connection" shared between validation and download.
|
|
77
|
+
#
|
|
78
|
+
# ssrf_filter 1.0.8's public `get`/`head` API has no response-streaming
|
|
79
|
+
# hook: `fetch_once` calls `Net::HTTP#request` with no block, which
|
|
80
|
+
# reads the entire response body into memory before returning; the
|
|
81
|
+
# block `get`/`head` accept is for customizing the *outgoing* request,
|
|
82
|
+
# not consuming the response. So we can't use `SsrfFilter.get` here
|
|
83
|
+
# without buffering the whole remote file in memory first.
|
|
84
|
+
#
|
|
85
|
+
# Reuse ssrf_filter's resolver and IP policy, then pin the validated
|
|
86
|
+
# address with Net::HTTP#ipaddr=. Keeping the original hostname on
|
|
87
|
+
# the HTTP connection preserves native TLS SNI and certificate
|
|
88
|
+
# verification without relying on ssrf_filter's TLS monkeypatches.
|
|
89
|
+
def download_http_source(file, uri, temp_file)
|
|
90
|
+
headers = Bulkrax::CloudSourceValidator.request_headers(file)
|
|
91
|
+
stream_download(uri, headers) { |response| handle_response(file, response, temp_file) }
|
|
92
|
+
rescue SsrfFilter::Error
|
|
93
|
+
# The validator's own HEAD check already ran, but DNS can change
|
|
94
|
+
# between that check and this GET — same rationale as
|
|
95
|
+
# Bulkrax::CloudSourceValidator#validate_http_scheme!, and the same
|
|
96
|
+
# static, class-agnostic message for the same reason.
|
|
97
|
+
raise Bulkrax::UnsafeCloudSourceError, 'blocked unsafe remote source'
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def handle_response(file, response, temp_file)
|
|
101
|
+
unless response.is_a?(Net::HTTPSuccess)
|
|
102
|
+
# Static on purpose — this becomes the importer's status message,
|
|
103
|
+
# not just an ops log; see Bulkrax::CloudSourceValidator for why
|
|
104
|
+
# status codes/response details aren't echoed into it. The status
|
|
105
|
+
# code itself is logged (not attacker-controlled, not a credential).
|
|
106
|
+
Rails.logger.error("Bulkrax: cloud file download failed for #{file['file_name']} with status #{response.code}")
|
|
107
|
+
raise Bulkrax::UnsafeCloudSourceError, 'download failed: non-success response'
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
File.open(temp_file, 'wb') { |io| response.read_body { |chunk| io.write(chunk) } }
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Resolves `uri`'s host, rejects it unless it has at least one public
|
|
114
|
+
# IP (reusing ssrf_filter's own resolver/blacklist — see above), and
|
|
115
|
+
# connects directly to that IP: with the environment proxy explicitly
|
|
116
|
+
# disabled (`Net::HTTP.new`'s `p_addr` defaults to `:ENV`, which would
|
|
117
|
+
# otherwise route through `http_proxy`/`https_proxy` and hand the
|
|
118
|
+
# pinned connection to a third party), and with the real hostname sent
|
|
119
|
+
# as the `Host` header and used for TLS SNI/certificate verification
|
|
120
|
+
# instead of the IP. Yields the live `Net::HTTPResponse` to the block
|
|
121
|
+
# *before* its body is read, so the caller can stream it.
|
|
122
|
+
def stream_download(uri, headers)
|
|
123
|
+
raise SsrfFilter::InvalidUriScheme, "URI scheme '#{uri.scheme}' not in whitelist" unless
|
|
124
|
+
SsrfFilter::DEFAULT_SCHEME_WHITELIST.include?(uri.scheme)
|
|
125
|
+
|
|
126
|
+
hostname = uri.hostname
|
|
127
|
+
ip_addresses = SsrfFilter::DEFAULT_RESOLVER.call(hostname)
|
|
128
|
+
raise SsrfFilter::UnresolvedHostname, "Could not resolve hostname '#{hostname}'" if ip_addresses.empty?
|
|
129
|
+
|
|
130
|
+
public_addresses = ip_addresses.reject { |ip| unsafe_ip_address?(ip) }
|
|
131
|
+
raise SsrfFilter::PrivateIPAddress, "Hostname '#{hostname}' has no public ip addresses" if public_addresses.empty?
|
|
132
|
+
|
|
133
|
+
request = Net::HTTP::Get.new(uri)
|
|
134
|
+
request['host'] = host_header(hostname, uri)
|
|
135
|
+
headers.each { |key, value| request[key] = value }
|
|
136
|
+
|
|
137
|
+
http = Net::HTTP.new(hostname, uri.port, nil)
|
|
138
|
+
http.ipaddr = public_addresses.sample.to_s
|
|
139
|
+
http.use_ssl = (uri.scheme == 'https')
|
|
140
|
+
http.start { |connection| connection.request(request) { |response| yield response } }
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Mirrors `SsrfFilter.unsafe_ip_address?`, but against its public
|
|
144
|
+
# `IPV4_BLACKLIST`/`IPV6_BLACKLIST` constants directly rather than
|
|
145
|
+
# calling that private method — we only ever check single resolved
|
|
146
|
+
# addresses here (never a CIDR range), so the masked-range guard it
|
|
147
|
+
# also carries doesn't apply.
|
|
148
|
+
def unsafe_ip_address?(ip_address)
|
|
149
|
+
return SsrfFilter::IPV4_BLACKLIST.any? { |range| range.include?(ip_address) } if ip_address.ipv4?
|
|
150
|
+
return SsrfFilter::IPV6_BLACKLIST.any? { |range| range.include?(ip_address) } if ip_address.ipv6?
|
|
151
|
+
|
|
152
|
+
true
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# Matches `SsrfFilter.host_header`: the `Host` header should carry the
|
|
156
|
+
# real hostname (not the pinned IP we actually connect to), with the
|
|
157
|
+
# port suffixed only when it isn't the scheme's default.
|
|
158
|
+
def host_header(hostname, uri)
|
|
159
|
+
if (uri.port == 80 && uri.scheme == 'http') || (uri.port == 443 && uri.scheme == 'https')
|
|
160
|
+
hostname
|
|
161
|
+
else
|
|
162
|
+
"#{hostname}:#{uri.port}"
|
|
27
163
|
end
|
|
28
|
-
Rails.logger.info "Download complete: #{file['url']} to #{target_file}"
|
|
29
164
|
end
|
|
30
165
|
end
|
|
31
166
|
end
|
|
@@ -472,27 +472,27 @@ module Bulkrax
|
|
|
472
472
|
# Zip Slip preflight — reject entries whose names are obviously unsafe
|
|
473
473
|
# (absolute paths, `..` segments) before we touch the filesystem.
|
|
474
474
|
# {#safe_extract_path} is the final line of defense; this check just
|
|
475
|
-
# fails fast with a clear message.
|
|
475
|
+
# fails fast with a clear message. Delegates to {Bulkrax::SafePath} so
|
|
476
|
+
# cloud-file destinations and sources share the same rules.
|
|
476
477
|
#
|
|
477
478
|
# @raise [Bulkrax::UnzipError] if the entry name is unsafe
|
|
478
|
-
def reject_unsafe_entry!(name)
|
|
479
|
-
|
|
480
|
-
|
|
479
|
+
def self.reject_unsafe_entry!(name)
|
|
480
|
+
Bulkrax::SafePath.reject_unsafe_entry!(name)
|
|
481
|
+
rescue Bulkrax::UnsafePathError
|
|
482
|
+
raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: name.dup.force_encoding(Encoding::UTF_8).scrub)
|
|
481
483
|
end
|
|
482
484
|
|
|
483
485
|
# Zip Slip chokepoint. Resolves `relative_dest` against `dest_dir` and
|
|
484
486
|
# returns the absolute destination path — but only if it stays inside
|
|
485
|
-
# `dest_dir
|
|
486
|
-
# own path with `File.join`, so the
|
|
487
|
-
# construction.
|
|
487
|
+
# `dest_dir`, including through any symlink. Callers must use this
|
|
488
|
+
# value rather than building their own path with `File.join`, so the
|
|
489
|
+
# path returned is always safe by construction.
|
|
488
490
|
#
|
|
489
491
|
# @return [String] absolute destination path, validated to be inside `dest_dir`
|
|
490
492
|
# @raise [Bulkrax::UnzipError] if `relative_dest` escapes `dest_dir`
|
|
491
493
|
def safe_extract_path(dest_dir, relative_dest)
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
return dest_path if dest_path == expanded_dest_dir
|
|
495
|
-
return dest_path if dest_path.start_with?("#{expanded_dest_dir}#{File::SEPARATOR}")
|
|
494
|
+
Bulkrax::SafePath.confine!(dest_dir, relative_dest)
|
|
495
|
+
rescue Bulkrax::UnsafePathError
|
|
496
496
|
raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: relative_dest)
|
|
497
497
|
end
|
|
498
498
|
|