bulkrax 8.2.4 → 8.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/javascripts/bulkrax/bulkrax.js +9 -7
- data/app/assets/javascripts/bulkrax/importers.js.erb +134 -99
- data/app/controllers/bulkrax/importers_controller.rb +28 -112
- data/app/controllers/concerns/bulkrax/datatables_behavior.rb +1 -1
- data/app/factories/bulkrax/object_factory.rb +1 -4
- data/app/jobs/bulkrax/download_cloud_file_job.rb +21 -156
- data/app/jobs/bulkrax/export_work_job.rb +1 -1
- data/app/models/bulkrax/status.rb +1 -1
- data/app/models/concerns/bulkrax/export_behavior.rb +4 -8
- data/app/parsers/bulkrax/application_parser.rb +2 -25
- data/app/parsers/bulkrax/bagit_parser.rb +3 -10
- data/app/parsers/bulkrax/csv_parser.rb +17 -51
- data/app/parsers/bulkrax/parser_export_record_set.rb +1 -1
- data/app/views/bulkrax/importers/_bagit_fields.html.erb +10 -3
- data/app/views/bulkrax/importers/_csv_fields.html.erb +13 -9
- data/app/views/bulkrax/importers/_file_uploader.html.erb +37 -0
- data/app/views/bulkrax/importers/_xml_fields.html.erb +10 -4
- data/config/locales/bulkrax.en.yml +0 -3
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +0 -1
- data/lib/tasks/bulkrax_tasks.rake +1 -1
- metadata +9 -32
- data/app/errors/bulkrax/unsafe_cloud_source_error.rb +0 -11
- data/app/errors/bulkrax/unsafe_path_error.rb +0 -8
- data/app/parsers/bulkrax/application_parser.rb.orig +0 -545
- data/app/services/bulkrax/cloud_source_validator.rb +0 -123
- data/app/services/bulkrax/safe_path.rb +0 -142
- data/db/migrate/20241203010707_entry_error_denormalization.rb +0 -7
- data/db/migrate/20241205212513_faster_first_entry.rb +0 -7
|
@@ -1,123 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Bulkrax
|
|
4
|
-
# Validates a cloud-file source (the Hash a user selected via
|
|
5
|
-
# browse-everything, or supplied directly in importer params) before
|
|
6
|
-
# Bulkrax downloads it. Destination confinement (see {Bulkrax::SafePath})
|
|
7
|
-
# stops a malicious filename from writing outside the importer's
|
|
8
|
-
# workspace; this validator stops a malicious `url` from reading a local
|
|
9
|
-
# file outside the configured filesystem-provider root, or from reaching
|
|
10
|
-
# a private/internal network over HTTP(S) (SSRF).
|
|
11
|
-
#
|
|
12
|
-
# Called twice by design: once before enqueueing, so an obviously unsafe
|
|
13
|
-
# batch never reaches a job queue, and again inside the download job,
|
|
14
|
-
# so a legacy or tampered job argument can't bypass validation.
|
|
15
|
-
class CloudSourceValidator
|
|
16
|
-
LOCAL_FILE_HOSTS = ['', 'localhost'].freeze
|
|
17
|
-
|
|
18
|
-
# Lightweight preflight for a single cloud-file selection: checks the
|
|
19
|
-
# raw file name and the source's safety, without resolving anything
|
|
20
|
-
# against an importer's workspace. Shared by CsvParser#plan_cloud_file,
|
|
21
|
-
# BagitParser#retrieve_cloud_files, and ImportersController's pre-save
|
|
22
|
-
# check, so a selection is judged unsafe the same way whether or not an
|
|
23
|
-
# importer has been persisted yet.
|
|
24
|
-
#
|
|
25
|
-
# @param file [Hash] a single cloud-file spec with at least `url` and `file_name` keys
|
|
26
|
-
# @raise [Bulkrax::UnsafePathError] if the raw file name is unsafe
|
|
27
|
-
# @raise [Bulkrax::UnsafeCloudSourceError] if the source isn't safe to download
|
|
28
|
-
# @return [true]
|
|
29
|
-
def self.preflight!(file)
|
|
30
|
-
name = file['file_name'] || file[:file_name]
|
|
31
|
-
raise Bulkrax::UnsafePathError, 'cloud file name must not be blank' if name.blank?
|
|
32
|
-
Bulkrax::SafePath.reject_unsafe_entry!(name)
|
|
33
|
-
validate!(file)
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
# @param file [Hash] a single cloud-file spec with at least a `url` key
|
|
37
|
-
# @raise [Bulkrax::UnsafeCloudSourceError] if the source isn't safe to download
|
|
38
|
-
# @return [true]
|
|
39
|
-
def self.validate!(file)
|
|
40
|
-
url = file['url'] || file[:url]
|
|
41
|
-
raise Bulkrax::UnsafeCloudSourceError, 'the cloud file is missing a url' if url.blank?
|
|
42
|
-
|
|
43
|
-
uri = parse_uri(url)
|
|
44
|
-
case uri.scheme
|
|
45
|
-
when 'file'
|
|
46
|
-
validate_file_scheme!(uri)
|
|
47
|
-
when 'http', 'https'
|
|
48
|
-
validate_http_scheme!(uri, file)
|
|
49
|
-
else
|
|
50
|
-
raise Bulkrax::UnsafeCloudSourceError, 'unsupported source scheme'
|
|
51
|
-
end
|
|
52
|
-
true
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
def self.parse_uri(url)
|
|
56
|
-
Addressable::URI.parse(url.to_s)
|
|
57
|
-
rescue Addressable::URI::InvalidURIError
|
|
58
|
-
raise Bulkrax::UnsafeCloudSourceError, 'the cloud file url is malformed'
|
|
59
|
-
end
|
|
60
|
-
private_class_method :parse_uri
|
|
61
|
-
|
|
62
|
-
def self.validate_file_scheme!(uri)
|
|
63
|
-
raise Bulkrax::UnsafeCloudSourceError, 'a file: source must not specify a remote host' unless LOCAL_FILE_HOSTS.include?(uri.host.to_s)
|
|
64
|
-
|
|
65
|
-
home = file_system_home
|
|
66
|
-
raise Bulkrax::UnsafeCloudSourceError, 'the filesystem provider is not configured' if home.blank?
|
|
67
|
-
|
|
68
|
-
decoded_path = URI::DEFAULT_PARSER.unescape(uri.path.to_s)
|
|
69
|
-
Bulkrax::SafePath.confine!(home, decoded_path, require_file: true, allow_absolute: true)
|
|
70
|
-
rescue Bulkrax::UnsafePathError => e
|
|
71
|
-
raise Bulkrax::UnsafeCloudSourceError, e.message
|
|
72
|
-
end
|
|
73
|
-
private_class_method :validate_file_scheme!
|
|
74
|
-
|
|
75
|
-
# Enforced at connection time, not by a DNS preflight: SsrfFilter
|
|
76
|
-
# resolves the hostname, rejects private/loopback/link-local/metadata
|
|
77
|
-
# addresses, and then connects directly to the validated IP (with TLS
|
|
78
|
-
# verified against the original hostname), closing the gap a
|
|
79
|
-
# resolve-then-connect check would leave open to DNS rebinding.
|
|
80
|
-
# Redirects are disabled outright (rather than re-validated per hop)
|
|
81
|
-
# so a header carrying provider credentials is never resent to a
|
|
82
|
-
# different host.
|
|
83
|
-
def self.validate_http_scheme!(uri, file)
|
|
84
|
-
raise Bulkrax::UnsafeCloudSourceError, 'no remote cloud provider is configured' unless remote_provider_configured?
|
|
85
|
-
raise Bulkrax::UnsafeCloudSourceError, 'the cloud file url must include a host' if uri.host.blank?
|
|
86
|
-
raise Bulkrax::UnsafeCloudSourceError, 'the cloud file url must not include credentials' if uri.userinfo.present?
|
|
87
|
-
|
|
88
|
-
SsrfFilter.head(uri.to_s, headers: request_headers(file), max_redirects: 0)
|
|
89
|
-
rescue SsrfFilter::Error, SocketError, SystemCallError, Timeout::Error, OpenSSL::SSL::SSLError
|
|
90
|
-
# Static and class-agnostic on purpose: SsrfFilter's own exception
|
|
91
|
-
# message can include the resolved IP or hostname it rejected, and
|
|
92
|
-
# this error is shown to the importing user / written to the
|
|
93
|
-
# importer's status, not just an ops log.
|
|
94
|
-
raise Bulkrax::UnsafeCloudSourceError, 'blocked unsafe remote source'
|
|
95
|
-
end
|
|
96
|
-
private_class_method :validate_http_scheme!
|
|
97
|
-
|
|
98
|
-
# Merges the caller-supplied headers (used for provider auth) but
|
|
99
|
-
# strips any transport-control header a malicious selection could use
|
|
100
|
-
# to smuggle a different Host past SsrfFilter's own Host header.
|
|
101
|
-
def self.request_headers(file)
|
|
102
|
-
headers = {}
|
|
103
|
-
headers.merge!(file['headers'] || file[:headers] || {})
|
|
104
|
-
headers.merge!(file['auth_header'] || file[:auth_header] || {})
|
|
105
|
-
headers.reject { |key, _| key.to_s.casecmp('host').zero? }
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
def self.file_system_home
|
|
109
|
-
config = provider_config('file_system')
|
|
110
|
-
config && (config['home'] || config[:home])
|
|
111
|
-
end
|
|
112
|
-
|
|
113
|
-
def self.remote_provider_configured?
|
|
114
|
-
BrowseEverything.config.any? { |key, value| key.to_s != 'file_system' && value.present? }
|
|
115
|
-
end
|
|
116
|
-
|
|
117
|
-
def self.provider_config(name)
|
|
118
|
-
config = BrowseEverything.config
|
|
119
|
-
config[name] || config[name.to_sym]
|
|
120
|
-
end
|
|
121
|
-
private_class_method :provider_config
|
|
122
|
-
end
|
|
123
|
-
end
|
|
@@ -1,142 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Bulkrax
|
|
4
|
-
# Shared path-confinement checks used anywhere Bulkrax resolves a
|
|
5
|
-
# caller-supplied relative path against a trusted root directory: ZIP
|
|
6
|
-
# extraction, cloud-file destinations, and filesystem cloud sources.
|
|
7
|
-
#
|
|
8
|
-
# Lexical checks alone (rejecting `..` and absolute paths) don't catch a
|
|
9
|
-
# symlink planted inside the root that points back out of it, so
|
|
10
|
-
# {.confine!} also resolves the longest existing ancestor of the
|
|
11
|
-
# candidate path and checks *that* for escape.
|
|
12
|
-
#
|
|
13
|
-
# @raise [Bulkrax::UnsafePathError] from any public method, never a bare
|
|
14
|
-
# StandardError, so callers can rescue one error class.
|
|
15
|
-
class SafePath
|
|
16
|
-
# Built via Integer#chr, not a literal escape, so this source
|
|
17
|
-
# file never contains a raw NUL byte (which some tools,
|
|
18
|
-
# including `git diff`, treat as a signal to classify the
|
|
19
|
-
# whole file as binary).
|
|
20
|
-
NUL = 0.chr.freeze
|
|
21
|
-
|
|
22
|
-
# Preflight check for a raw, caller-supplied name — a ZIP entry name,
|
|
23
|
-
# or a cloud-file's uploaded filename — before any join, space
|
|
24
|
-
# normalization, or filesystem access. Callers must run this on the
|
|
25
|
-
# *raw* name: joining it under a prefix first (e.g. `File.join('files',
|
|
26
|
-
# name)`) can silently neutralize a leading `/`, which would defeat
|
|
27
|
-
# this check if run afterward. {.confine!} runs it again on the
|
|
28
|
-
# resolved candidate as the final authority.
|
|
29
|
-
#
|
|
30
|
-
# @param name [String] a raw entry/file name
|
|
31
|
-
# @raise [Bulkrax::UnsafePathError] if the name is an absolute path, has
|
|
32
|
-
# a `..` segment, or contains a NUL byte
|
|
33
|
-
def self.reject_unsafe_entry!(name)
|
|
34
|
-
name = name.to_s
|
|
35
|
-
raise Bulkrax::UnsafePathError, 'path must not contain a NUL byte' if name.include?(NUL)
|
|
36
|
-
raise Bulkrax::UnsafePathError, 'path must not contain a parent-directory reference' if name.split('/').include?('..')
|
|
37
|
-
raise Bulkrax::UnsafePathError, 'path must not be an absolute path' if name.start_with?('/')
|
|
38
|
-
end
|
|
39
|
-
|
|
40
|
-
# Resolves `candidate` against `root` and returns the absolute result,
|
|
41
|
-
# but only if it stays inside `root` once both symlinks and `..`
|
|
42
|
-
# segments are resolved.
|
|
43
|
-
#
|
|
44
|
-
# By default `candidate` is held to the same lexical rule as
|
|
45
|
-
# {.reject_unsafe_entry!}: no `..` segment and no leading `/`, even if
|
|
46
|
-
# the fully-resolved result would happen to land inside `root` — a
|
|
47
|
-
# destination filename that only *looks* safe after normalization is
|
|
48
|
-
# still rejected outright, per the plan, rather than silently
|
|
49
|
-
# corrected. The one legitimate exception is a `file:` *source* path
|
|
50
|
-
# (always given as an absolute path by the filesystem provider);
|
|
51
|
-
# pass `allow_absolute: true` for that case. `..` segments are never
|
|
52
|
-
# allowed, even then.
|
|
53
|
-
#
|
|
54
|
-
# @param root [String] trusted directory that the result must stay inside
|
|
55
|
-
# @param candidate [String] a relative (or, with `allow_absolute`, absolute) path to resolve
|
|
56
|
-
# @param require_file [Boolean] when true, reject a result equal to `root` itself
|
|
57
|
-
# @param allow_absolute [Boolean] when true, permit `candidate` to be an absolute path
|
|
58
|
-
# @return [String] absolute path, validated to be inside `root`
|
|
59
|
-
# @raise [Bulkrax::UnsafePathError] if `candidate` is blank, contains a
|
|
60
|
-
# NUL byte, has a `..` segment, is an absolute path (unless
|
|
61
|
-
# `allow_absolute`), escapes `root`, or escapes `root` via a symlink
|
|
62
|
-
def self.confine!(root, candidate, require_file: false, allow_absolute: false)
|
|
63
|
-
raise Bulkrax::UnsafePathError, 'path must not be blank' if candidate.blank?
|
|
64
|
-
|
|
65
|
-
candidate = candidate.to_s
|
|
66
|
-
if allow_absolute
|
|
67
|
-
raise Bulkrax::UnsafePathError, 'path must not contain a NUL byte' if candidate.include?(NUL)
|
|
68
|
-
raise Bulkrax::UnsafePathError, 'path must not contain a parent-directory reference' if candidate.split('/').include?('..')
|
|
69
|
-
else
|
|
70
|
-
reject_unsafe_entry!(candidate)
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
expanded_root = File.expand_path(root)
|
|
74
|
-
expanded_path = File.expand_path(candidate, expanded_root)
|
|
75
|
-
|
|
76
|
-
raise Bulkrax::UnsafePathError, 'path escapes the confined root' unless expanded_path == expanded_root || expanded_path.start_with?("#{expanded_root}#{File::SEPARATOR}")
|
|
77
|
-
raise Bulkrax::UnsafePathError, 'a file path is required' if require_file && expanded_path == expanded_root
|
|
78
|
-
|
|
79
|
-
reject_symlink_escape!(expanded_root, expanded_path)
|
|
80
|
-
expanded_path
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
# True if a filesystem entry exists at `path` — including a dangling
|
|
84
|
-
# symlink. File.exist? follows symlinks and reports `false` for one
|
|
85
|
-
# whose target doesn't exist yet, which is exactly the entry a caller
|
|
86
|
-
# is about to write through, so it can't be the only check here.
|
|
87
|
-
def self.present?(path)
|
|
88
|
-
File.exist?(path) || File.symlink?(path)
|
|
89
|
-
end
|
|
90
|
-
private_class_method :present?
|
|
91
|
-
|
|
92
|
-
# Resolves `path` to its real, final location, tolerating a dangling
|
|
93
|
-
# symlink (one whose target doesn't exist) by following the link
|
|
94
|
-
# lexically instead of raising the way File.realpath would. Bounded
|
|
95
|
-
# to 32 hops so a symlink cycle fails closed rather than hanging.
|
|
96
|
-
def self.resolve_fully(path)
|
|
97
|
-
current = path
|
|
98
|
-
32.times do
|
|
99
|
-
return File.realpath(current) if File.exist?(current)
|
|
100
|
-
return current unless File.symlink?(current)
|
|
101
|
-
current = File.expand_path(File.readlink(current), File.dirname(current))
|
|
102
|
-
end
|
|
103
|
-
raise Bulkrax::UnsafePathError, 'path has too many levels of symbolic links'
|
|
104
|
-
end
|
|
105
|
-
private_class_method :resolve_fully
|
|
106
|
-
|
|
107
|
-
# Resolves the longest existing-or-dangling-symlink ancestor of
|
|
108
|
-
# `expanded_path` (which may be `expanded_path` itself) and checks it
|
|
109
|
-
# for a symlink escape. Path components that aren't present at all
|
|
110
|
-
# (not even as a symlink) can't be symlinks, so this is the final
|
|
111
|
-
# line of defense without racing the filesystem. As the plan assumes,
|
|
112
|
-
# this is not a guard against a *concurrent* write to the same path —
|
|
113
|
-
# only against a symlink already planted in an application-controlled
|
|
114
|
-
# staging directory.
|
|
115
|
-
def self.reject_symlink_escape!(expanded_root, expanded_path)
|
|
116
|
-
real_root = present?(expanded_root) ? resolve_fully(expanded_root) : expanded_root
|
|
117
|
-
|
|
118
|
-
existing_ancestor = expanded_path
|
|
119
|
-
existing_ancestor = File.dirname(existing_ancestor) until present?(existing_ancestor) || existing_ancestor == File.dirname(existing_ancestor)
|
|
120
|
-
return unless present?(existing_ancestor)
|
|
121
|
-
|
|
122
|
-
real_existing = resolve_fully(existing_ancestor)
|
|
123
|
-
|
|
124
|
-
# existing_ancestor is normally at or below root (inside expanded_path's
|
|
125
|
-
# own subtree) and must resolve to somewhere inside real_root. But when
|
|
126
|
-
# root itself doesn't exist yet (e.g. a first-time extraction
|
|
127
|
-
# destination), the walk climbs past root to a shallower ancestor —
|
|
128
|
-
# the relationship is reversed, and it's real_root that must resolve
|
|
129
|
-
# to somewhere inside (or at) real_existing instead.
|
|
130
|
-
ancestor_within_root = existing_ancestor == expanded_root || existing_ancestor.start_with?("#{expanded_root}#{File::SEPARATOR}")
|
|
131
|
-
contained = if ancestor_within_root
|
|
132
|
-
real_existing == real_root || real_existing.start_with?("#{real_root}#{File::SEPARATOR}")
|
|
133
|
-
else
|
|
134
|
-
real_root == real_existing || real_root.start_with?("#{real_existing}#{File::SEPARATOR}")
|
|
135
|
-
end
|
|
136
|
-
return if contained
|
|
137
|
-
|
|
138
|
-
raise Bulkrax::UnsafePathError, 'path escapes the confined root via a symlink'
|
|
139
|
-
end
|
|
140
|
-
private_class_method :reject_symlink_escape!
|
|
141
|
-
end
|
|
142
|
-
end
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
class EntryErrorDenormalization < ActiveRecord::Migration[5.1]
|
|
2
|
-
def change
|
|
3
|
-
add_column :bulkrax_entries, :error_class, :string unless column_exists?(:bulkrax_entries, :error_class)
|
|
4
|
-
add_column :bulkrax_importers, :error_class, :string unless column_exists?(:bulkrax_importers, :error_class)
|
|
5
|
-
add_column :bulkrax_exporters, :error_class, :string unless column_exists?(:bulkrax_exporters, :error_class)
|
|
6
|
-
end
|
|
7
|
-
end
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
class FasterFirstEntry < ActiveRecord::Migration[5.2]
|
|
3
|
-
def change
|
|
4
|
-
add_index :bulkrax_entries, [:importerexporter_id, :importerexporter_type, :id], name: 'index_bulkrax_entries_on_importerexporter_id_type_and_id' unless index_exists?(:bulkrax_entries, [:importerexporter_id, :importerexporter_type, :id],
|
|
5
|
-
name: 'index_bulkrax_entries_on_importerexporter_id_type_and_id')
|
|
6
|
-
end
|
|
7
|
-
end
|