bulkrax 8.2.2 → 8.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/controllers/bulkrax/importers_controller.rb +115 -20
- data/app/errors/bulkrax/unsafe_cloud_source_error.rb +11 -0
- data/app/errors/bulkrax/unsafe_path_error.rb +8 -0
- data/app/jobs/bulkrax/download_cloud_file_job.rb +156 -21
- data/app/jobs/bulkrax/export_work_job.rb +1 -1
- data/app/parsers/bulkrax/application_parser.rb +25 -2
- data/app/parsers/bulkrax/application_parser.rb.orig +545 -0
- data/app/parsers/bulkrax/bagit_parser.rb +10 -3
- data/app/parsers/bulkrax/csv_parser.rb +46 -15
- data/app/services/bulkrax/cloud_source_validator.rb +123 -0
- data/app/services/bulkrax/safe_path.rb +142 -0
- data/config/locales/bulkrax.en.yml +3 -0
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +1 -0
- metadata +28 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 921dc741e247f2b7ebd31f9e1b00d95f7a5da6f42fc4106979bc2456ce52d39c
|
|
4
|
+
data.tar.gz: 9d595dddab1708878df613bc2b40315974af950273eb26fffbbfa6dcdd5e5958
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cc708da6667febae646836c3720b2a74e7b0eb6b4c0414b53a583ff54006be82a567906029d951c8172497df8538aaa25e964bd012beee3ade6d59df379cb12f
|
|
7
|
+
data.tar.gz: c6920e180a2eb093bbec5e29c508c124c57c77f5039c8352c69a1704169c5540e55276ad2d45429e0056ea36949b2dca7155da375d6da443b3e99608c1cc1e6e
|
|
@@ -78,6 +78,7 @@ module Bulkrax
|
|
|
78
78
|
|
|
79
79
|
# POST /importers
|
|
80
80
|
# rubocop:disable Metrics/MethodLength
|
|
81
|
+
# rubocop:disable Metrics/AbcSize
|
|
81
82
|
def create
|
|
82
83
|
# rubocop:disable Style/IfInsideElse
|
|
83
84
|
if api_request?
|
|
@@ -92,8 +93,15 @@ module Bulkrax
|
|
|
92
93
|
# the following line is needed to handle updating remote files of a FileSet
|
|
93
94
|
# on a new import otherwise it only gets updated during the update path
|
|
94
95
|
@importer.parser_fields['update_files'] = true if params[:commit] == 'Create and Import'
|
|
96
|
+
|
|
97
|
+
# Reject an unsafe cloud selection before the importer is ever
|
|
98
|
+
# persisted, rather than after save when a row would already exist.
|
|
99
|
+
preflight_cloud_files!(cloud_files)
|
|
100
|
+
|
|
95
101
|
if @importer.save
|
|
96
|
-
files_for_import(file, cloud_files)
|
|
102
|
+
return render_create_failure if files_for_import(file, cloud_files) && !@importer.save
|
|
103
|
+
dispatch_cloud_downloads!
|
|
104
|
+
|
|
97
105
|
if params[:commit] == 'Create and Import'
|
|
98
106
|
Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id)
|
|
99
107
|
render_request('Importer was successfully created and import has been queued.')
|
|
@@ -104,14 +112,13 @@ module Bulkrax
|
|
|
104
112
|
render_request('Importer was successfully created.')
|
|
105
113
|
end
|
|
106
114
|
else
|
|
107
|
-
|
|
108
|
-
json_response('create', :unprocessable_entity)
|
|
109
|
-
else
|
|
110
|
-
render :new
|
|
111
|
-
end
|
|
115
|
+
render_create_failure
|
|
112
116
|
end
|
|
113
117
|
# rubocop:enable Style/IfInsideElse
|
|
118
|
+
rescue Bulkrax::UnsafeCloudSourceError, Bulkrax::UnsafePathError => e
|
|
119
|
+
reject_unsafe_cloud_files(e)
|
|
114
120
|
end
|
|
121
|
+
# rubocop:enable Metrics/AbcSize
|
|
115
122
|
|
|
116
123
|
# PATCH/PUT /importers/1
|
|
117
124
|
# # @todo refactor so as to not need to disable rubocop
|
|
@@ -127,11 +134,19 @@ module Bulkrax
|
|
|
127
134
|
# Skipped during a continue
|
|
128
135
|
field_mapping_params if params[:importer][:parser_fields].present?
|
|
129
136
|
|
|
130
|
-
|
|
131
|
-
|
|
137
|
+
# Assign the proposed attributes in memory first, so an unsafe cloud
|
|
138
|
+
# selection is rejected before anything is persisted; the rescue
|
|
139
|
+
# below then leaves the importer's persisted attributes untouched.
|
|
140
|
+
@importer.assign_attributes(importer_params)
|
|
141
|
+
preflight_cloud_files!(cloud_files)
|
|
142
|
+
|
|
143
|
+
if @importer.save
|
|
144
|
+
return render_update_failure if files_for_import(file, cloud_files) && !@importer.save
|
|
145
|
+
return render_update_failure if params[:commit] != 'Update Importer' && !set_files_parser_fields
|
|
146
|
+
dispatch_cloud_downloads!
|
|
147
|
+
|
|
132
148
|
# do not perform the import
|
|
133
149
|
unless params[:commit] == 'Update Importer'
|
|
134
|
-
set_files_parser_fields
|
|
135
150
|
Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id, update_harvest)
|
|
136
151
|
end
|
|
137
152
|
if api_request?
|
|
@@ -140,12 +155,10 @@ module Bulkrax
|
|
|
140
155
|
redirect_to importers_path, notice: 'Importer was successfully updated.'
|
|
141
156
|
end
|
|
142
157
|
else
|
|
143
|
-
|
|
144
|
-
json_response('update', :unprocessable_entity, 'Something went wrong.')
|
|
145
|
-
else
|
|
146
|
-
render :edit
|
|
147
|
-
end
|
|
158
|
+
render_update_failure
|
|
148
159
|
end
|
|
160
|
+
rescue Bulkrax::UnsafeCloudSourceError, Bulkrax::UnsafePathError => e
|
|
161
|
+
reject_unsafe_cloud_files(e)
|
|
149
162
|
end
|
|
150
163
|
# rubocop:enable all
|
|
151
164
|
# rubocop:enable Metrics/MethodLength
|
|
@@ -218,18 +231,68 @@ module Bulkrax
|
|
|
218
231
|
|
|
219
232
|
private
|
|
220
233
|
|
|
234
|
+
# Validates every selected cloud-file entry — raw name and source
|
|
235
|
+
# safety — before the importer is created or updated at all. Must not
|
|
236
|
+
# touch the importer's workspace (no path_for_import, no directory
|
|
237
|
+
# creation, no download): for #create that workspace doesn't exist
|
|
238
|
+
# yet, and for #update it must not be assumed safe to touch before the
|
|
239
|
+
# proposed attributes are known to be acceptable.
|
|
240
|
+
#
|
|
241
|
+
# @param cloud_files [Hash, nil] cloud file specs from browse-everything
|
|
242
|
+
# @raise [Bulkrax::UnsafePathError, Bulkrax::UnsafeCloudSourceError] on the first unsafe entry
|
|
243
|
+
def preflight_cloud_files!(cloud_files)
|
|
244
|
+
return if cloud_files.blank?
|
|
245
|
+
|
|
246
|
+
cloud_files.values.each { |file| Bulkrax::CloudSourceValidator.preflight!(file) }
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
# Assigns files/cloud selections onto the importer record's in-memory
|
|
250
|
+
# parser fields, but does not save: the caller owns the second save,
|
|
251
|
+
# so it can check its result and avoid rendering success or enqueueing
|
|
252
|
+
# an import on failure.
|
|
253
|
+
#
|
|
254
|
+
# @return [Boolean] true if any parser field was touched and a second save is needed
|
|
221
255
|
def files_for_import(file, cloud_files)
|
|
222
|
-
|
|
256
|
+
@cloud_download_plan = nil
|
|
257
|
+
return false if file.blank? && cloud_files.blank?
|
|
258
|
+
|
|
223
259
|
@importer[:parser_fields]['import_file_path'] = @importer.parser.write_import_file(file) if file.present?
|
|
224
260
|
if cloud_files.present?
|
|
225
261
|
@importer[:parser_fields]['cloud_file_paths'] = cloud_files
|
|
262
|
+
assign_cloud_file_paths(cloud_files)
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
true
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# For a parser that separates planning from dispatch (currently only
|
|
269
|
+
# CsvParser), resolves and validates the whole batch and records the
|
|
270
|
+
# target paths, deferring enqueue until {#dispatch_cloud_downloads!} is
|
|
271
|
+
# called after the controller persists those paths. Other parsers
|
|
272
|
+
# (e.g. BagitParser) already complete cloud retrieval synchronously
|
|
273
|
+
# and return the resolved path directly, so there's nothing left to
|
|
274
|
+
# dispatch.
|
|
275
|
+
def assign_cloud_file_paths(cloud_files)
|
|
276
|
+
parser = @importer.parser
|
|
277
|
+
if parser.respond_to?(:plan_cloud_files)
|
|
278
|
+
@cloud_download_plan = parser.plan_cloud_files(cloud_files)
|
|
279
|
+
@importer[:parser_fields]['original_file_paths'] =
|
|
280
|
+
@cloud_download_plan.map { |_file, _relative_filename, target_file| target_file }
|
|
281
|
+
else
|
|
226
282
|
# For BagIt, there will only be one bag, so we get the file_path back and set import_file_path
|
|
227
|
-
|
|
228
|
-
# and we expect the import_file_path to be set already
|
|
229
|
-
target = @importer.parser.retrieve_cloud_files(cloud_files, @importer)
|
|
283
|
+
target = parser.retrieve_cloud_files(cloud_files, @importer)
|
|
230
284
|
@importer[:parser_fields]['import_file_path'] = target if target.present?
|
|
231
285
|
end
|
|
232
|
-
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
# Dispatches a plan prepared by {#assign_cloud_file_paths}, if any.
|
|
289
|
+
# Called by the controller only after it has successfully persisted
|
|
290
|
+
# the resolved paths, so a download is never enqueued for a path the
|
|
291
|
+
# importer doesn't actually have on record.
|
|
292
|
+
def dispatch_cloud_downloads!
|
|
293
|
+
return if @cloud_download_plan.blank?
|
|
294
|
+
|
|
295
|
+
@importer.parser.dispatch_cloud_files!(@cloud_download_plan, @importer)
|
|
233
296
|
end
|
|
234
297
|
|
|
235
298
|
# Use callbacks to share common setup or constraints between actions.
|
|
@@ -315,6 +378,36 @@ module Bulkrax
|
|
|
315
378
|
'text/csv'
|
|
316
379
|
end
|
|
317
380
|
|
|
381
|
+
# Reports an unsafe cloud-file batch without leaking the error's class
|
|
382
|
+
# or backtrace (which could include a provider auth header) to the
|
|
383
|
+
# response; `e.message` is always one of our own static, credential-free
|
|
384
|
+
# strings (see Bulkrax::CloudSourceValidator / Bulkrax::SafePath).
|
|
385
|
+
def reject_unsafe_cloud_files(e)
|
|
386
|
+
Rails.logger.error("Bulkrax: rejected unsafe cloud file batch for importer #{@importer&.id}: #{e.class}: #{e.message}")
|
|
387
|
+
if api_request?
|
|
388
|
+
json_response('create', :unprocessable_entity, e.message)
|
|
389
|
+
else
|
|
390
|
+
flash[:error] = e.message
|
|
391
|
+
redirect_back fallback_location: importers_path
|
|
392
|
+
end
|
|
393
|
+
end
|
|
394
|
+
|
|
395
|
+
def render_create_failure
|
|
396
|
+
if api_request?
|
|
397
|
+
json_response('create', :unprocessable_entity)
|
|
398
|
+
else
|
|
399
|
+
render :new
|
|
400
|
+
end
|
|
401
|
+
end
|
|
402
|
+
|
|
403
|
+
def render_update_failure
|
|
404
|
+
if api_request?
|
|
405
|
+
json_response('update', :unprocessable_entity, 'Something went wrong.')
|
|
406
|
+
else
|
|
407
|
+
render :edit
|
|
408
|
+
end
|
|
409
|
+
end
|
|
410
|
+
|
|
318
411
|
def render_request(message, validate_only = false)
|
|
319
412
|
if api_request?
|
|
320
413
|
json_response('create', :created, message)
|
|
@@ -330,6 +423,8 @@ module Bulkrax
|
|
|
330
423
|
params[:commit] == 'Update and Harvest Updated Items'
|
|
331
424
|
end
|
|
332
425
|
|
|
426
|
+
# @return [Boolean] the result of the save, so the caller can treat a
|
|
427
|
+
# failure here like an ordinary validation failure
|
|
333
428
|
def set_files_parser_fields
|
|
334
429
|
@importer.parser_fields['update_files'] =
|
|
335
430
|
@importer.parser_fields['replace_files'] =
|
|
@@ -342,7 +437,7 @@ module Bulkrax
|
|
|
342
437
|
elsif params[:commit] == 'Remove and Rerun'
|
|
343
438
|
@importer.parser_fields['remove_and_rerun'] = true
|
|
344
439
|
elsif params[:commit] == 'Update and Harvest Updated Items'
|
|
345
|
-
return
|
|
440
|
+
return true
|
|
346
441
|
else
|
|
347
442
|
@importer.parser_fields['metadata_only'] = true
|
|
348
443
|
end
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
# Raised by {Bulkrax::CloudSourceValidator} when a cloud-file source
|
|
5
|
+
# (the `url` a user selected via browse-everything, or supplied directly
|
|
6
|
+
# in importer params) can't be safely downloaded: a scheme other than
|
|
7
|
+
# `file`/`http`/`https`, a `file:` source outside the configured
|
|
8
|
+
# filesystem-provider root, or an HTTP(S) source that resolves to a
|
|
9
|
+
# private, loopback, or link-local/metadata address.
|
|
10
|
+
class UnsafeCloudSourceError < StandardError; end
|
|
11
|
+
end
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
# Raised by {Bulkrax::SafePath} when a path would escape its configured
|
|
5
|
+
# root, whether that root is a ZIP extraction directory, an importer's
|
|
6
|
+
# trusted workspace, or a configured filesystem-provider home.
|
|
7
|
+
class UnsafePathError < StandardError; end
|
|
8
|
+
end
|
|
@@ -1,31 +1,166 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
+
|
|
2
3
|
module Bulkrax
|
|
3
4
|
class DownloadCloudFileJob < ApplicationJob
|
|
4
5
|
queue_as Bulkrax.config.ingest_queue_name
|
|
5
6
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
#
|
|
9
|
-
#
|
|
7
|
+
# Retrieve a cloud file and write it to the importer's workspace.
|
|
8
|
+
#
|
|
9
|
+
# `importer_id:` and `relative_filename:` are required keywords: the
|
|
10
|
+
# destination is always recomputed here, confined to this importer's
|
|
11
|
+
# own workspace, rather than trusted from a serialized job argument.
|
|
12
|
+
# `target_file` is a deprecated positional placeholder, kept only so
|
|
13
|
+
# that a pre-upgrade call shape (`file`, `target_file`) still binds
|
|
14
|
+
# without Ruby raising its own ArgumentError — it is never read, even
|
|
15
|
+
# when supplied alongside otherwise-valid keywords, so it can't be
|
|
16
|
+
# used to redirect the destination. A call made this way is missing
|
|
17
|
+
# the required keywords, so it raises below rather than running.
|
|
18
|
+
#
|
|
19
|
+
# Note: if using the file system provider, the mounted directory in
|
|
10
20
|
# browse_everything MUST be shared by web and worker servers
|
|
11
|
-
def perform(file, target_file)
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
21
|
+
def perform(file, target_file = nil, importer_id: nil, relative_filename: nil)
|
|
22
|
+
raise Bulkrax::UnsafePathError, I18n.t('bulkrax.importer.browse_everything.errors.missing_destination_context') if importer_id.blank? || relative_filename.blank?
|
|
23
|
+
|
|
24
|
+
# `file` and `relative_filename` are independent arguments and may
|
|
25
|
+
# not agree (a tampered or replayed job call could supply either
|
|
26
|
+
# one separately) — check the raw file_name directly rather than
|
|
27
|
+
# assuming it was already validated by whatever enqueued this job.
|
|
28
|
+
Bulkrax::SafePath.reject_unsafe_entry!(file['file_name'])
|
|
29
|
+
|
|
30
|
+
importer = Bulkrax::Importer.find(importer_id)
|
|
31
|
+
target_file = Bulkrax::SafePath.confine!(importer.parser.path_for_import, relative_filename, require_file: true)
|
|
32
|
+
|
|
33
|
+
# Re-validate the source here too: this job may run on a different
|
|
34
|
+
# process/host than the one that enqueued it, and could be invoked
|
|
35
|
+
# directly with a tampered or stale `file` argument.
|
|
36
|
+
Bulkrax::CloudSourceValidator.validate!(file)
|
|
37
|
+
|
|
38
|
+
download(file, target_file)
|
|
39
|
+
Rails.logger.info "Bulkrax: cloud file download complete for #{file['file_name']}"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
# Downloads through a temp file in the *same* directory as
|
|
45
|
+
# `target_file` (so the final move is same-filesystem and atomic),
|
|
46
|
+
# named with a random suffix rather than anything derived from
|
|
47
|
+
# caller-controlled input, and removed on any failure. This means a
|
|
48
|
+
# failed or non-success download can never truncate or corrupt an
|
|
49
|
+
# already-good destination file from a previous successful import.
|
|
50
|
+
def download(file, target_file)
|
|
51
|
+
temp_file = "#{target_file}.#{SecureRandom.hex(8)}.part"
|
|
52
|
+
uri = Addressable::URI.parse(file['url'].to_s)
|
|
53
|
+
if uri.scheme == 'file'
|
|
54
|
+
download_file_source(uri, temp_file)
|
|
55
|
+
else
|
|
56
|
+
download_http_source(file, URI.parse(file['url'].to_s), temp_file)
|
|
57
|
+
end
|
|
58
|
+
FileUtils.mv(temp_file, target_file)
|
|
59
|
+
ensure
|
|
60
|
+
FileUtils.rm_f(temp_file) if temp_file && File.exist?(temp_file)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# The source was already confined to the configured filesystem
|
|
64
|
+
# provider's root by {Bulkrax::CloudSourceValidator}.
|
|
65
|
+
def download_file_source(uri, temp_file)
|
|
66
|
+
source_path = URI::DEFAULT_PARSER.unescape(uri.path.to_s)
|
|
67
|
+
IO.copy_stream(source_path, temp_file)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Resolved, IP-pinned, redirect-free, and streamed straight to disk in
|
|
71
|
+
# bounded-size chunks — rather than handing the URL back to a transport
|
|
72
|
+
# (browse-everything's Retriever) that re-resolves the hostname with no
|
|
73
|
+
# SSRF protection at all. This is a *separate* connection from the one
|
|
74
|
+
# {Bulkrax::CloudSourceValidator}'s HEAD check made; each independently
|
|
75
|
+
# applies the same validation at its own connection time, so there is
|
|
76
|
+
# no single "pinned connection" shared between validation and download.
|
|
77
|
+
#
|
|
78
|
+
# ssrf_filter 1.0.8's public `get`/`head` API has no response-streaming
|
|
79
|
+
# hook: `fetch_once` calls `Net::HTTP#request` with no block, which
|
|
80
|
+
# reads the entire response body into memory before returning; the
|
|
81
|
+
# block `get`/`head` accept is for customizing the *outgoing* request,
|
|
82
|
+
# not consuming the response. So we can't use `SsrfFilter.get` here
|
|
83
|
+
# without buffering the whole remote file in memory first.
|
|
84
|
+
#
|
|
85
|
+
# Reuse ssrf_filter's resolver and IP policy, then pin the validated
|
|
86
|
+
# address with Net::HTTP#ipaddr=. Keeping the original hostname on
|
|
87
|
+
# the HTTP connection preserves native TLS SNI and certificate
|
|
88
|
+
# verification without relying on ssrf_filter's TLS monkeypatches.
|
|
89
|
+
def download_http_source(file, uri, temp_file)
|
|
90
|
+
headers = Bulkrax::CloudSourceValidator.request_headers(file)
|
|
91
|
+
stream_download(uri, headers) { |response| handle_response(file, response, temp_file) }
|
|
92
|
+
rescue SsrfFilter::Error
|
|
93
|
+
# The validator's own HEAD check already ran, but DNS can change
|
|
94
|
+
# between that check and this GET — same rationale as
|
|
95
|
+
# Bulkrax::CloudSourceValidator#validate_http_scheme!, and the same
|
|
96
|
+
# static, class-agnostic message for the same reason.
|
|
97
|
+
raise Bulkrax::UnsafeCloudSourceError, 'blocked unsafe remote source'
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def handle_response(file, response, temp_file)
|
|
101
|
+
unless response.is_a?(Net::HTTPSuccess)
|
|
102
|
+
# Static on purpose — this becomes the importer's status message,
|
|
103
|
+
# not just an ops log; see Bulkrax::CloudSourceValidator for why
|
|
104
|
+
# status codes/response details aren't echoed into it. The status
|
|
105
|
+
# code itself is logged (not attacker-controlled, not a credential).
|
|
106
|
+
Rails.logger.error("Bulkrax: cloud file download failed for #{file['file_name']} with status #{response.code}")
|
|
107
|
+
raise Bulkrax::UnsafeCloudSourceError, 'download failed: non-success response'
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
File.open(temp_file, 'wb') { |io| response.read_body { |chunk| io.write(chunk) } }
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Resolves `uri`'s host, rejects it unless it has at least one public
|
|
114
|
+
# IP (reusing ssrf_filter's own resolver/blacklist — see above), and
|
|
115
|
+
# connects directly to that IP: with the environment proxy explicitly
|
|
116
|
+
# disabled (`Net::HTTP.new`'s `p_addr` defaults to `:ENV`, which would
|
|
117
|
+
# otherwise route through `http_proxy`/`https_proxy` and hand the
|
|
118
|
+
# pinned connection to a third party), and with the real hostname sent
|
|
119
|
+
# as the `Host` header and used for TLS SNI/certificate verification
|
|
120
|
+
# instead of the IP. Yields the live `Net::HTTPResponse` to the block
|
|
121
|
+
# *before* its body is read, so the caller can stream it.
|
|
122
|
+
def stream_download(uri, headers)
|
|
123
|
+
raise SsrfFilter::InvalidUriScheme, "URI scheme '#{uri.scheme}' not in whitelist" unless
|
|
124
|
+
SsrfFilter::DEFAULT_SCHEME_WHITELIST.include?(uri.scheme)
|
|
125
|
+
|
|
126
|
+
hostname = uri.hostname
|
|
127
|
+
ip_addresses = SsrfFilter::DEFAULT_RESOLVER.call(hostname)
|
|
128
|
+
raise SsrfFilter::UnresolvedHostname, "Could not resolve hostname '#{hostname}'" if ip_addresses.empty?
|
|
129
|
+
|
|
130
|
+
public_addresses = ip_addresses.reject { |ip| unsafe_ip_address?(ip) }
|
|
131
|
+
raise SsrfFilter::PrivateIPAddress, "Hostname '#{hostname}' has no public ip addresses" if public_addresses.empty?
|
|
132
|
+
|
|
133
|
+
request = Net::HTTP::Get.new(uri)
|
|
134
|
+
request['host'] = host_header(hostname, uri)
|
|
135
|
+
headers.each { |key, value| request[key] = value }
|
|
136
|
+
|
|
137
|
+
http = Net::HTTP.new(hostname, uri.port, nil)
|
|
138
|
+
http.ipaddr = public_addresses.sample.to_s
|
|
139
|
+
http.use_ssl = (uri.scheme == 'https')
|
|
140
|
+
http.start { |connection| connection.request(request) { |response| yield response } }
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Mirrors `SsrfFilter.unsafe_ip_address?`, but against its public
|
|
144
|
+
# `IPV4_BLACKLIST`/`IPV6_BLACKLIST` constants directly rather than
|
|
145
|
+
# calling that private method — we only ever check single resolved
|
|
146
|
+
# addresses here (never a CIDR range), so the masked-range guard it
|
|
147
|
+
# also carries doesn't apply.
|
|
148
|
+
def unsafe_ip_address?(ip_address)
|
|
149
|
+
return SsrfFilter::IPV4_BLACKLIST.any? { |range| range.include?(ip_address) } if ip_address.ipv4?
|
|
150
|
+
return SsrfFilter::IPV6_BLACKLIST.any? { |range| range.include?(ip_address) } if ip_address.ipv6?
|
|
151
|
+
|
|
152
|
+
true
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# Matches `SsrfFilter.host_header`: the `Host` header should carry the
|
|
156
|
+
# real hostname (not the pinned IP we actually connect to), with the
|
|
157
|
+
# port suffixed only when it isn't the scheme's default.
|
|
158
|
+
def host_header(hostname, uri)
|
|
159
|
+
if (uri.port == 80 && uri.scheme == 'http') || (uri.port == 443 && uri.scheme == 'https')
|
|
160
|
+
hostname
|
|
161
|
+
else
|
|
162
|
+
"#{hostname}:#{uri.port}"
|
|
27
163
|
end
|
|
28
|
-
Rails.logger.info "Download complete: #{file['url']} to #{target_file}"
|
|
29
164
|
end
|
|
30
165
|
end
|
|
31
166
|
end
|
|
@@ -26,7 +26,7 @@ module Bulkrax
|
|
|
26
26
|
end
|
|
27
27
|
# rubocop:enable Rails/SkipsModelValidations
|
|
28
28
|
end
|
|
29
|
-
return entry if exporter_run.enqueued_records.positive?
|
|
29
|
+
return entry if exporter_run.reload.enqueued_records.positive?
|
|
30
30
|
|
|
31
31
|
if exporter_run.failed_records.positive?
|
|
32
32
|
exporter_run.exporter.set_status_info('Complete (with failures)')
|
|
@@ -430,15 +430,38 @@ module Bulkrax
|
|
|
430
430
|
def unzip(file_to_unzip)
|
|
431
431
|
return untar(file_to_unzip) if file_to_unzip.end_with?('.tar.gz')
|
|
432
432
|
|
|
433
|
+
dest_dir = importer_unzip_path
|
|
433
434
|
Zip::File.open(file_to_unzip) do |zip_file|
|
|
434
435
|
zip_file.each do |entry|
|
|
435
|
-
|
|
436
|
+
# Zip Slip: reject/confine via the same Bulkrax::SafePath used
|
|
437
|
+
# for cloud-file destinations, rather than a bare File.join of
|
|
438
|
+
# attacker-controlled entry.name.
|
|
439
|
+
entry_path = Bulkrax::SafePath.confine!(dest_dir, entry.name)
|
|
436
440
|
FileUtils.mkdir_p(File.dirname(entry_path))
|
|
437
|
-
|
|
441
|
+
next if File.exist?(entry_path)
|
|
442
|
+
|
|
443
|
+
extract_zip_entry(zip_file, entry, dest_dir, entry.name, entry_path)
|
|
438
444
|
end
|
|
439
445
|
end
|
|
440
446
|
end
|
|
441
447
|
|
|
448
|
+
# rubyzip 2.x: extract(entry, absolute_dest_path)
|
|
449
|
+
# rubyzip 3.x: extract(entry, relative_name, destination_directory: dir)
|
|
450
|
+
#
|
|
451
|
+
# entry_path (from Bulkrax::SafePath.confine!) is always an absolute
|
|
452
|
+
# path; rubyzip 3.x's 2-positional-arg #extract treats its second
|
|
453
|
+
# argument as a name relative to the current directory rather than a
|
|
454
|
+
# destination path, which silently re-joins an already-absolute path
|
|
455
|
+
# onto the cwd. The keyword form is the only one both versions can be
|
|
456
|
+
# trusted to interpret as "this absolute/relative destination".
|
|
457
|
+
def extract_zip_entry(zip_file, entry, dest_dir, relative_name, dest_path)
|
|
458
|
+
if zip_file.method(:extract).arity == 2
|
|
459
|
+
zip_file.extract(entry, dest_path)
|
|
460
|
+
else
|
|
461
|
+
zip_file.extract(entry, relative_name, destination_directory: dest_dir)
|
|
462
|
+
end
|
|
463
|
+
end
|
|
464
|
+
|
|
442
465
|
def untar(file_to_untar)
|
|
443
466
|
Dir.mkdir(importer_unzip_path) unless File.directory?(importer_unzip_path)
|
|
444
467
|
command = "tar -xzf #{Shellwords.escape(file_to_untar)} -C #{Shellwords.escape(importer_unzip_path)}"
|