bulkrax 9.5.1 → 10.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +49 -0
- data/app/controllers/bulkrax/application_controller.rb +83 -0
- data/app/controllers/bulkrax/entries_controller.rb +16 -13
- data/app/controllers/bulkrax/exporters_controller.rb +21 -25
- data/app/controllers/bulkrax/guided_imports_controller.rb +4 -8
- data/app/controllers/bulkrax/importers_controller.rb +84 -46
- data/app/controllers/concerns/bulkrax/datatables_behavior.rb +18 -12
- data/app/controllers/concerns/bulkrax/importer_file_handler.rb +55 -9
- data/app/errors/bulkrax/unsafe_cloud_source_error.rb +11 -0
- data/app/errors/bulkrax/unsafe_path_error.rb +8 -0
- data/app/factories/bulkrax/object_factory.rb +2 -2
- data/app/factories/bulkrax/valkyrie_object_factory.rb +29 -17
- data/app/helpers/bulkrax/validation_helper.rb +1 -1
- data/app/jobs/bulkrax/create_relationships_job.rb +1 -1
- data/app/jobs/bulkrax/download_cloud_file_job.rb +156 -21
- data/app/jobs/bulkrax/import_collection_job.rb +9 -2
- data/app/matchers/bulkrax/application_matcher.rb +5 -2
- data/app/models/bulkrax/csv_entry.rb +1 -1
- data/app/models/bulkrax/current.rb +7 -0
- data/app/models/bulkrax/export_scope.rb +21 -0
- data/app/models/bulkrax/exporter.rb +2 -2
- data/app/models/bulkrax/importer.rb +1 -1
- data/app/models/concerns/bulkrax/ability_behavior.rb +139 -0
- data/app/models/concerns/bulkrax/has_matchers.rb +1 -1
- data/app/models/concerns/bulkrax/import_behavior.rb +3 -3
- data/app/parsers/bulkrax/application_parser.rb +11 -11
- data/app/parsers/bulkrax/application_parser.rb.orig +545 -0
- data/app/parsers/bulkrax/bagit_parser.rb +10 -3
- data/app/parsers/bulkrax/csv_parser.rb +59 -27
- data/app/parsers/bulkrax/oai_dc_parser.rb +1 -1
- data/app/parsers/bulkrax/parser_export_record_set.rb +26 -8
- data/app/parsers/concerns/bulkrax/csv_parser/csv_template_generation.rb +1 -1
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation.rb +1 -2
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_helpers.rb +18 -1
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_hierarchy.rb +2 -2
- data/app/search_builders/bulkrax/export_search_builder.rb +9 -0
- data/app/services/bulkrax/cloud_source_validator.rb +123 -0
- data/app/services/bulkrax/csv_template/column_builder.rb +2 -2
- data/app/services/bulkrax/csv_template/mapping_manager.rb +9 -7
- data/app/services/bulkrax/csv_template/schema_analyzer.rb +1 -5
- data/app/services/bulkrax/safe_path.rb +129 -0
- data/config/locales/bulkrax.en.yml +2 -0
- data/lib/bulkrax/engine.rb +1 -1
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +77 -10
- data/lib/generators/bulkrax/install_generator.rb +22 -10
- data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb +1 -1
- metadata +58 -14
- data/app/factories/bulkrax/valkyrize-hyku.code-workspace +0 -19
- data/lib/wings/custom_queries/find_by_source_identifier.rb +0 -41
|
@@ -71,7 +71,7 @@ module Bulkrax
|
|
|
71
71
|
metadata = {
|
|
72
72
|
visibility: 'open'
|
|
73
73
|
}
|
|
74
|
-
metadata[:collection_type_gid] = Hyrax::CollectionType.find_or_create_default_collection_type.to_global_id.to_s if
|
|
74
|
+
metadata[:collection_type_gid] = Hyrax::CollectionType.find_or_create_default_collection_type.to_global_id.to_s if Bulkrax.hyrax_loaded?
|
|
75
75
|
|
|
76
76
|
collections.each_with_index do |set, index|
|
|
77
77
|
next unless collection_name == 'all' || collection_name == set.spec
|
|
@@ -120,7 +120,10 @@ module Bulkrax
|
|
|
120
120
|
#
|
|
121
121
|
# No sense attempting to query for more than the limit.
|
|
122
122
|
def query_kwargs
|
|
123
|
-
{ fl: "id,#{Bulkrax.solr_key_for_member_file_ids}", method: :post, rows: row_limit }
|
|
123
|
+
base = { fl: "id,#{Bulkrax.solr_key_for_member_file_ids}", method: :post, rows: row_limit }
|
|
124
|
+
filters = permission_filters
|
|
125
|
+
return base if filters.empty?
|
|
126
|
+
base.merge(fq: filters)
|
|
124
127
|
end
|
|
125
128
|
|
|
126
129
|
# If we have a limit, we need not query beyond that limit
|
|
@@ -175,10 +178,25 @@ module Bulkrax
|
|
|
175
178
|
@file_sets ||= ParserExportRecordSet.in_batches(candidate_file_set_ids) do |batch_of_ids|
|
|
176
179
|
fsq = "has_model_ssim:\"#{Bulkrax.file_model_internal_resource.demodulize}\" AND id:(\"" + batch_of_ids.join('" OR "') + "\")"
|
|
177
180
|
fsq += extra_filters if extra_filters.present?
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
)
|
|
181
|
+
fq = permission_filters
|
|
182
|
+
kwargs = { fl: "id", method: :post, rows: batch_of_ids.size }
|
|
183
|
+
kwargs[:fq] = fq unless fq.empty?
|
|
184
|
+
Bulkrax.object_factory.query(fsq, **kwargs)
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def exporting_user
|
|
189
|
+
importerexporter.user || Bulkrax.fallback_user_for_importer_exporter_processing
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def permission_filters
|
|
193
|
+
@permission_filters ||= begin
|
|
194
|
+
ability = Ability.new(exporting_user)
|
|
195
|
+
return [] if ability.can_admin_exporters?
|
|
196
|
+
|
|
197
|
+
scope = Bulkrax::ExportScope.new(ability)
|
|
198
|
+
builder = Bulkrax::ExportSearchBuilder.new(scope)
|
|
199
|
+
builder.to_h.fetch(:fq, [])
|
|
182
200
|
end
|
|
183
201
|
end
|
|
184
202
|
|
|
@@ -246,7 +264,7 @@ module Bulkrax
|
|
|
246
264
|
Bulkrax.object_factory.query(
|
|
247
265
|
extra_filters.to_s,
|
|
248
266
|
**query_kwargs.merge(
|
|
249
|
-
fq: [
|
|
267
|
+
fq: (query_kwargs[:fq] || []) + [
|
|
250
268
|
%(#{solr_name(work_identifier)}:("#{ids.join('" OR "')}")),
|
|
251
269
|
"has_model_ssim:(#{Bulkrax.curation_concern_internal_resources.join(' OR ')})"
|
|
252
270
|
],
|
|
@@ -261,7 +279,7 @@ module Bulkrax
|
|
|
261
279
|
Bulkrax.object_factory.query(
|
|
262
280
|
"has_model_ssim:#{Bulkrax.collection_model_internal_resource} #{extra_filters}",
|
|
263
281
|
**query_kwargs.merge(
|
|
264
|
-
fq: [
|
|
282
|
+
fq: (query_kwargs[:fq] || []) + [
|
|
265
283
|
%(#{solr_name(work_identifier)}:("#{ids.join('" OR "')}")),
|
|
266
284
|
"has_model_ssim:#{Bulkrax.collection_model_internal_resource}"
|
|
267
285
|
],
|
|
@@ -280,7 +298,7 @@ module Bulkrax
|
|
|
280
298
|
Bulkrax.object_factory.query(
|
|
281
299
|
extra_filters,
|
|
282
300
|
**query_kwargs.merge(
|
|
283
|
-
fq: [
|
|
301
|
+
fq: (query_kwargs[:fq] || []) + [
|
|
284
302
|
%(#{solr_name(work_identifier)}:("#{ids.join('" OR "')}")),
|
|
285
303
|
"has_model_ssim:#{Bulkrax.file_model_internal_resource}"
|
|
286
304
|
],
|
|
@@ -14,7 +14,7 @@ module Bulkrax
|
|
|
14
14
|
# @param args [Hash] Additional arguments passed to output method (e.g., file_path)
|
|
15
15
|
# @return [String] File path (for 'file' output) or CSV string (for 'csv_string' output)
|
|
16
16
|
def generate_template(models: [], output: 'file', admin_set_id: nil, **args)
|
|
17
|
-
raise NameError, "Hyrax is not defined" unless
|
|
17
|
+
raise NameError, "Hyrax is not defined" unless Bulkrax.hyrax_loaded?
|
|
18
18
|
TemplateContext.new(models: models, admin_set_id: admin_set_id).send("to_#{output}", **args)
|
|
19
19
|
end
|
|
20
20
|
end
|
|
@@ -71,8 +71,7 @@ module Bulkrax
|
|
|
71
71
|
file_key = resolve_validation_key(mapping_manager, key: 'file', default: :file)
|
|
72
72
|
|
|
73
73
|
csv_data = parse_validation_rows(raw_csv, source_id_key, parent_key, children_key, file_key)
|
|
74
|
-
all_models = csv_data
|
|
75
|
-
all_models |= [Bulkrax.default_work_type] if Bulkrax.default_work_type.present?
|
|
74
|
+
all_models = models_to_validate(csv_data, headers)
|
|
76
75
|
field_analyzer = CsvTemplate::FieldAnalyzer.new(mappings, admin_set_id)
|
|
77
76
|
field_metadata = build_validation_field_metadata(all_models, field_analyzer)
|
|
78
77
|
|
|
@@ -64,7 +64,7 @@ module Bulkrax
|
|
|
64
64
|
Rails.logger.error("CsvParser.validate_csv: error building valid headers – #{e.message}")
|
|
65
65
|
standard = %w[model source_identifier parents children file]
|
|
66
66
|
model_fields = field_metadata.values.flat_map { |m| m[:properties] }
|
|
67
|
-
|
|
67
|
+
model_fields = model_fields.map { |prop| mapping_manager.key_to_mapped_column(prop) }
|
|
68
68
|
(standard + model_fields).uniq
|
|
69
69
|
end
|
|
70
70
|
|
|
@@ -129,6 +129,23 @@ module Bulkrax
|
|
|
129
129
|
all_models.each { |model| missing_required << { model: model, field: source_id_key.to_s } }
|
|
130
130
|
end
|
|
131
131
|
|
|
132
|
+
# Adding the default work type unconditionally makes every sheet answer
|
|
133
|
+
# for that model's required fields, so a collection-only sheet gets asked
|
|
134
|
+
# for properties belonging to a work type it never mentions.
|
|
135
|
+
def models_to_validate(csv_data, headers)
|
|
136
|
+
row_models = csv_data.map { |r| r[:model].to_s }.reject(&:blank?).uniq
|
|
137
|
+
return row_models if Bulkrax.default_work_type.blank?
|
|
138
|
+
return row_models unless default_work_type_needed?(csv_data, headers)
|
|
139
|
+
|
|
140
|
+
row_models | [Bulkrax.default_work_type]
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def default_work_type_needed?(csv_data, headers)
|
|
144
|
+
return true unless headers.map(&:to_s).include?('model')
|
|
145
|
+
|
|
146
|
+
csv_data.empty? || csv_data.any? { |r| r[:model].blank? }
|
|
147
|
+
end
|
|
148
|
+
|
|
132
149
|
# Adds a file-level notice when the model column is absent or every row has a blank
|
|
133
150
|
# model value, indicating that the default work type will be used for all rows.
|
|
134
151
|
# When this notice is present the per-row default_work_type_used warnings are
|
|
@@ -36,9 +36,9 @@ module Bulkrax
|
|
|
36
36
|
model_str = item[:model].to_s
|
|
37
37
|
|
|
38
38
|
opts = { type: nil, find_record: find_record, parent: parent_split_pattern, child: child_split_pattern }
|
|
39
|
-
if
|
|
39
|
+
if Bulkrax.collection_model_name?(model_str)
|
|
40
40
|
collections << build_item_hash(item, child_to_parents, all_ids, opts.merge(type: 'collection'))
|
|
41
|
-
elsif
|
|
41
|
+
elsif Bulkrax.file_model_name?(model_str)
|
|
42
42
|
file_sets << { id: item_id, title: item[:raw_row]['title'] || item_id, type: 'file_set' }
|
|
43
43
|
else
|
|
44
44
|
works << build_item_hash(item, child_to_parents, all_ids, opts.merge(type: 'work'))
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
# Validates a cloud-file source (the Hash a user selected via
|
|
5
|
+
# browse-everything, or supplied directly in importer params) before
|
|
6
|
+
# Bulkrax downloads it. Destination confinement (see {Bulkrax::SafePath})
|
|
7
|
+
# stops a malicious filename from writing outside the importer's
|
|
8
|
+
# workspace; this validator stops a malicious `url` from reading a local
|
|
9
|
+
# file outside the configured filesystem-provider root, or from reaching
|
|
10
|
+
# a private/internal network over HTTP(S) (SSRF).
|
|
11
|
+
#
|
|
12
|
+
# Called twice by design: once before enqueueing, so an obviously unsafe
|
|
13
|
+
# batch never reaches a job queue, and again inside the download job,
|
|
14
|
+
# so a legacy or tampered job argument can't bypass validation.
|
|
15
|
+
class CloudSourceValidator
|
|
16
|
+
LOCAL_FILE_HOSTS = ['', 'localhost'].freeze
|
|
17
|
+
|
|
18
|
+
# Lightweight preflight for a single cloud-file selection: checks the
|
|
19
|
+
# raw file name and the source's safety, without resolving anything
|
|
20
|
+
# against an importer's workspace. Shared by CsvParser#plan_cloud_file,
|
|
21
|
+
# BagitParser#retrieve_cloud_files, and ImportersController's pre-save
|
|
22
|
+
# check, so a selection is judged unsafe the same way whether or not an
|
|
23
|
+
# importer has been persisted yet.
|
|
24
|
+
#
|
|
25
|
+
# @param file [Hash] a single cloud-file spec with at least `url` and `file_name` keys
|
|
26
|
+
# @raise [Bulkrax::UnsafePathError] if the raw file name is unsafe
|
|
27
|
+
# @raise [Bulkrax::UnsafeCloudSourceError] if the source isn't safe to download
|
|
28
|
+
# @return [true]
|
|
29
|
+
def self.preflight!(file)
|
|
30
|
+
name = file['file_name'] || file[:file_name]
|
|
31
|
+
raise Bulkrax::UnsafePathError, 'cloud file name must not be blank' if name.blank?
|
|
32
|
+
Bulkrax::SafePath.reject_unsafe_entry!(name)
|
|
33
|
+
validate!(file)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# @param file [Hash] a single cloud-file spec with at least a `url` key
|
|
37
|
+
# @raise [Bulkrax::UnsafeCloudSourceError] if the source isn't safe to download
|
|
38
|
+
# @return [true]
|
|
39
|
+
def self.validate!(file)
|
|
40
|
+
url = file['url'] || file[:url]
|
|
41
|
+
raise Bulkrax::UnsafeCloudSourceError, 'the cloud file is missing a url' if url.blank?
|
|
42
|
+
|
|
43
|
+
uri = parse_uri(url)
|
|
44
|
+
case uri.scheme
|
|
45
|
+
when 'file'
|
|
46
|
+
validate_file_scheme!(uri)
|
|
47
|
+
when 'http', 'https'
|
|
48
|
+
validate_http_scheme!(uri, file)
|
|
49
|
+
else
|
|
50
|
+
raise Bulkrax::UnsafeCloudSourceError, 'unsupported source scheme'
|
|
51
|
+
end
|
|
52
|
+
true
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def self.parse_uri(url)
|
|
56
|
+
Addressable::URI.parse(url.to_s)
|
|
57
|
+
rescue Addressable::URI::InvalidURIError
|
|
58
|
+
raise Bulkrax::UnsafeCloudSourceError, 'the cloud file url is malformed'
|
|
59
|
+
end
|
|
60
|
+
private_class_method :parse_uri
|
|
61
|
+
|
|
62
|
+
def self.validate_file_scheme!(uri)
|
|
63
|
+
raise Bulkrax::UnsafeCloudSourceError, 'a file: source must not specify a remote host' unless LOCAL_FILE_HOSTS.include?(uri.host.to_s)
|
|
64
|
+
|
|
65
|
+
home = file_system_home
|
|
66
|
+
raise Bulkrax::UnsafeCloudSourceError, 'the filesystem provider is not configured' if home.blank?
|
|
67
|
+
|
|
68
|
+
decoded_path = URI::DEFAULT_PARSER.unescape(uri.path.to_s)
|
|
69
|
+
Bulkrax::SafePath.confine!(home, decoded_path, require_file: true, allow_absolute: true)
|
|
70
|
+
rescue Bulkrax::UnsafePathError => e
|
|
71
|
+
raise Bulkrax::UnsafeCloudSourceError, e.message
|
|
72
|
+
end
|
|
73
|
+
private_class_method :validate_file_scheme!
|
|
74
|
+
|
|
75
|
+
# Enforced at connection time, not by a DNS preflight: SsrfFilter
|
|
76
|
+
# resolves the hostname, rejects private/loopback/link-local/metadata
|
|
77
|
+
# addresses, and then connects directly to the validated IP (with TLS
|
|
78
|
+
# verified against the original hostname), closing the gap a
|
|
79
|
+
# resolve-then-connect check would leave open to DNS rebinding.
|
|
80
|
+
# Redirects are disabled outright (rather than re-validated per hop)
|
|
81
|
+
# so a header carrying provider credentials is never resent to a
|
|
82
|
+
# different host.
|
|
83
|
+
def self.validate_http_scheme!(uri, file)
|
|
84
|
+
raise Bulkrax::UnsafeCloudSourceError, 'no remote cloud provider is configured' unless remote_provider_configured?
|
|
85
|
+
raise Bulkrax::UnsafeCloudSourceError, 'the cloud file url must include a host' if uri.host.blank?
|
|
86
|
+
raise Bulkrax::UnsafeCloudSourceError, 'the cloud file url must not include credentials' if uri.userinfo.present?
|
|
87
|
+
|
|
88
|
+
SsrfFilter.head(uri.to_s, headers: request_headers(file), max_redirects: 0)
|
|
89
|
+
rescue SsrfFilter::Error, SocketError, SystemCallError, Timeout::Error, OpenSSL::SSL::SSLError
|
|
90
|
+
# Static and class-agnostic on purpose: SsrfFilter's own exception
|
|
91
|
+
# message can include the resolved IP or hostname it rejected, and
|
|
92
|
+
# this error is shown to the importing user / written to the
|
|
93
|
+
# importer's status, not just an ops log.
|
|
94
|
+
raise Bulkrax::UnsafeCloudSourceError, 'blocked unsafe remote source'
|
|
95
|
+
end
|
|
96
|
+
private_class_method :validate_http_scheme!
|
|
97
|
+
|
|
98
|
+
# Merges the caller-supplied headers (used for provider auth) but
|
|
99
|
+
# strips any transport-control header a malicious selection could use
|
|
100
|
+
# to smuggle a different Host past SsrfFilter's own Host header.
|
|
101
|
+
def self.request_headers(file)
|
|
102
|
+
headers = {}
|
|
103
|
+
headers.merge!(file['headers'] || file[:headers] || {})
|
|
104
|
+
headers.merge!(file['auth_header'] || file[:auth_header] || {})
|
|
105
|
+
headers.reject { |key, _| key.to_s.casecmp('host').zero? }
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def self.file_system_home
|
|
109
|
+
config = provider_config('file_system')
|
|
110
|
+
config && (config['home'] || config[:home])
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def self.remote_provider_configured?
|
|
114
|
+
BrowseEverything.config.any? { |key, value| key.to_s != 'file_system' && value.present? }
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def self.provider_config(name)
|
|
118
|
+
config = BrowseEverything.config
|
|
119
|
+
config[name] || config[name.to_sym]
|
|
120
|
+
end
|
|
121
|
+
private_class_method :provider_config
|
|
122
|
+
end
|
|
123
|
+
end
|
|
@@ -43,8 +43,8 @@ module Bulkrax
|
|
|
43
43
|
|
|
44
44
|
# When a property is the target of one or more `object:` field mappings,
|
|
45
45
|
# emit each of those mappings' `from:` columns (e.g. redirect_path,
|
|
46
|
-
#
|
|
47
|
-
#
|
|
46
|
+
# redirect_display_url) rather than the bare property name (redirects).
|
|
47
|
+
# Otherwise fall back to the standard 1:1 mapping.
|
|
48
48
|
def columns_for_property(property)
|
|
49
49
|
nested = @service.mapping_manager.object_columns_for(property)
|
|
50
50
|
return nested if nested.any?
|
|
@@ -35,16 +35,18 @@ module Bulkrax
|
|
|
35
35
|
|
|
36
36
|
# Returns the column names that target a given object name via the
|
|
37
37
|
# `object:` field-mapping pattern. The template generator uses this to
|
|
38
|
-
# emit the per-child columns (e.g. redirect_path,
|
|
39
|
-
#
|
|
40
|
-
#
|
|
41
|
-
#
|
|
42
|
-
#
|
|
38
|
+
# emit the per-child columns (e.g. redirect_path, redirect_display_url)
|
|
39
|
+
# instead of the bare property name (redirects). Numbering is
|
|
40
|
+
# intentionally omitted — the template shows the column shape once;
|
|
41
|
+
# CSV rows can repeat the column with numeric suffixes (e.g.
|
|
42
|
+
# redirect_path_1, redirect_path_2) at import time.
|
|
43
|
+
# An import accepts every `from:` alias, but offering them all here would
|
|
44
|
+
# put two columns for the same field on the template.
|
|
43
45
|
def object_columns_for(object_name)
|
|
44
46
|
@mappings
|
|
45
47
|
.select { |_k, v| v.is_a?(Hash) && v["object"] == object_name }
|
|
46
|
-
.
|
|
47
|
-
.
|
|
48
|
+
.keys
|
|
49
|
+
.map { |key| key_to_mapped_column(key) }
|
|
48
50
|
.uniq
|
|
49
51
|
end
|
|
50
52
|
|
|
@@ -36,11 +36,7 @@ module Bulkrax
|
|
|
36
36
|
def load_schema
|
|
37
37
|
return nil unless @klass.respond_to?(:schema)
|
|
38
38
|
|
|
39
|
-
|
|
40
|
-
Hyrax.schema_for(klass: @klass, admin_set_id: @admin_set_id)
|
|
41
|
-
else
|
|
42
|
-
@klass.new.singleton_class.schema || @klass.schema
|
|
43
|
-
end
|
|
39
|
+
Bulkrax::ValkyrieObjectFactory.cached_schema_for(klass: @klass, admin_set_id: @admin_set_id)
|
|
44
40
|
rescue StandardError
|
|
45
41
|
nil
|
|
46
42
|
end
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
# Shared path-confinement checks used anywhere Bulkrax resolves a
|
|
5
|
+
# caller-supplied relative path against a trusted root directory: ZIP
|
|
6
|
+
# extraction, cloud-file destinations, and filesystem cloud sources.
|
|
7
|
+
#
|
|
8
|
+
# Lexical checks alone (rejecting `..` and absolute paths) don't catch a
|
|
9
|
+
# symlink planted inside the root that points back out of it, so
|
|
10
|
+
# {.confine!} also resolves the longest existing ancestor of the
|
|
11
|
+
# candidate path and checks *that* for escape.
|
|
12
|
+
#
|
|
13
|
+
# @raise [Bulkrax::UnsafePathError] from any public method, never a bare
|
|
14
|
+
# StandardError, so callers can rescue one error class.
|
|
15
|
+
class SafePath
|
|
16
|
+
# Built via Integer#chr, not a literal escape, so this source
|
|
17
|
+
# file never contains a raw NUL byte (which some tools,
|
|
18
|
+
# including `git diff`, treat as a signal to classify the
|
|
19
|
+
# whole file as binary).
|
|
20
|
+
NUL = 0.chr.freeze
|
|
21
|
+
|
|
22
|
+
# Preflight check for a raw, caller-supplied name — a ZIP entry name,
|
|
23
|
+
# or a cloud-file's uploaded filename — before any join, space
|
|
24
|
+
# normalization, or filesystem access. Callers must run this on the
|
|
25
|
+
# *raw* name: joining it under a prefix first (e.g. `File.join('files',
|
|
26
|
+
# name)`) can silently neutralize a leading `/`, which would defeat
|
|
27
|
+
# this check if run afterward. {.confine!} runs it again on the
|
|
28
|
+
# resolved candidate as the final authority.
|
|
29
|
+
#
|
|
30
|
+
# @param name [String] a raw entry/file name
|
|
31
|
+
# @raise [Bulkrax::UnsafePathError] if the name is an absolute path, has
|
|
32
|
+
# a `..` segment, or contains a NUL byte
|
|
33
|
+
def self.reject_unsafe_entry!(name)
|
|
34
|
+
name = name.to_s
|
|
35
|
+
raise Bulkrax::UnsafePathError, 'path must not contain a NUL byte' if name.include?(NUL)
|
|
36
|
+
raise Bulkrax::UnsafePathError, 'path must not contain a parent-directory reference' if name.split('/').include?('..')
|
|
37
|
+
raise Bulkrax::UnsafePathError, 'path must not be an absolute path' if name.start_with?('/')
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Resolves `candidate` against `root` and returns the absolute result,
|
|
41
|
+
# but only if it stays inside `root` once both symlinks and `..`
|
|
42
|
+
# segments are resolved.
|
|
43
|
+
#
|
|
44
|
+
# By default `candidate` is held to the same lexical rule as
|
|
45
|
+
# {.reject_unsafe_entry!}: no `..` segment and no leading `/`, even if
|
|
46
|
+
# the fully-resolved result would happen to land inside `root` — a
|
|
47
|
+
# destination filename that only *looks* safe after normalization is
|
|
48
|
+
# still rejected outright, per the plan, rather than silently
|
|
49
|
+
# corrected. The one legitimate exception is a `file:` *source* path
|
|
50
|
+
# (always given as an absolute path by the filesystem provider);
|
|
51
|
+
# pass `allow_absolute: true` for that case. `..` segments are never
|
|
52
|
+
# allowed, even then.
|
|
53
|
+
#
|
|
54
|
+
# @param root [String] trusted directory that the result must stay inside
|
|
55
|
+
# @param candidate [String] a relative (or, with `allow_absolute`, absolute) path to resolve
|
|
56
|
+
# @param require_file [Boolean] when true, reject a result equal to `root` itself
|
|
57
|
+
# @param allow_absolute [Boolean] when true, permit `candidate` to be an absolute path
|
|
58
|
+
# @return [String] absolute path, validated to be inside `root`
|
|
59
|
+
# @raise [Bulkrax::UnsafePathError] if `candidate` is blank, contains a
|
|
60
|
+
# NUL byte, has a `..` segment, is an absolute path (unless
|
|
61
|
+
# `allow_absolute`), escapes `root`, or escapes `root` via a symlink
|
|
62
|
+
def self.confine!(root, candidate, require_file: false, allow_absolute: false)
|
|
63
|
+
raise Bulkrax::UnsafePathError, 'path must not be blank' if candidate.blank?
|
|
64
|
+
|
|
65
|
+
candidate = candidate.to_s
|
|
66
|
+
if allow_absolute
|
|
67
|
+
raise Bulkrax::UnsafePathError, 'path must not contain a NUL byte' if candidate.include?(NUL)
|
|
68
|
+
raise Bulkrax::UnsafePathError, 'path must not contain a parent-directory reference' if candidate.split('/').include?('..')
|
|
69
|
+
else
|
|
70
|
+
reject_unsafe_entry!(candidate)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
expanded_root = File.expand_path(root)
|
|
74
|
+
expanded_path = File.expand_path(candidate, expanded_root)
|
|
75
|
+
|
|
76
|
+
raise Bulkrax::UnsafePathError, 'path escapes the confined root' unless expanded_path == expanded_root || expanded_path.start_with?("#{expanded_root}#{File::SEPARATOR}")
|
|
77
|
+
raise Bulkrax::UnsafePathError, 'a file path is required' if require_file && expanded_path == expanded_root
|
|
78
|
+
|
|
79
|
+
reject_symlink_escape!(expanded_root, expanded_path)
|
|
80
|
+
expanded_path
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# True if a filesystem entry exists at `path` — including a dangling
|
|
84
|
+
# symlink. File.exist? follows symlinks and reports `false` for one
|
|
85
|
+
# whose target doesn't exist yet, which is exactly the entry a caller
|
|
86
|
+
# is about to write through, so it can't be the only check here.
|
|
87
|
+
def self.present?(path)
|
|
88
|
+
File.exist?(path) || File.symlink?(path)
|
|
89
|
+
end
|
|
90
|
+
private_class_method :present?
|
|
91
|
+
|
|
92
|
+
# Resolves `path` to its real, final location, tolerating a dangling
|
|
93
|
+
# symlink (one whose target doesn't exist) by following the link
|
|
94
|
+
# lexically instead of raising the way File.realpath would. Bounded
|
|
95
|
+
# to 32 hops so a symlink cycle fails closed rather than hanging.
|
|
96
|
+
def self.resolve_fully(path)
|
|
97
|
+
current = path
|
|
98
|
+
32.times do
|
|
99
|
+
return File.realpath(current) if File.exist?(current)
|
|
100
|
+
return current unless File.symlink?(current)
|
|
101
|
+
current = File.expand_path(File.readlink(current), File.dirname(current))
|
|
102
|
+
end
|
|
103
|
+
raise Bulkrax::UnsafePathError, 'path has too many levels of symbolic links'
|
|
104
|
+
end
|
|
105
|
+
private_class_method :resolve_fully
|
|
106
|
+
|
|
107
|
+
# Resolves the longest existing-or-dangling-symlink ancestor of
|
|
108
|
+
# `expanded_path` (which may be `expanded_path` itself) and checks it
|
|
109
|
+
# for a symlink escape. Path components that aren't present at all
|
|
110
|
+
# (not even as a symlink) can't be symlinks, so this is the final
|
|
111
|
+
# line of defense without racing the filesystem. As the plan assumes,
|
|
112
|
+
# this is not a guard against a *concurrent* write to the same path —
|
|
113
|
+
# only against a symlink already planted in an application-controlled
|
|
114
|
+
# staging directory.
|
|
115
|
+
def self.reject_symlink_escape!(expanded_root, expanded_path)
|
|
116
|
+
real_root = present?(expanded_root) ? resolve_fully(expanded_root) : expanded_root
|
|
117
|
+
|
|
118
|
+
existing_ancestor = expanded_path
|
|
119
|
+
existing_ancestor = File.dirname(existing_ancestor) until present?(existing_ancestor) || existing_ancestor == File.dirname(existing_ancestor)
|
|
120
|
+
return unless present?(existing_ancestor)
|
|
121
|
+
|
|
122
|
+
real_existing = resolve_fully(existing_ancestor)
|
|
123
|
+
return if real_existing == real_root || real_existing.start_with?("#{real_root}#{File::SEPARATOR}")
|
|
124
|
+
|
|
125
|
+
raise Bulkrax::UnsafePathError, 'path escapes the confined root via a symlink'
|
|
126
|
+
end
|
|
127
|
+
private_class_method :reject_symlink_escape!
|
|
128
|
+
end
|
|
129
|
+
end
|
|
@@ -91,6 +91,8 @@ en:
|
|
|
91
91
|
public: Public
|
|
92
92
|
browse_everything:
|
|
93
93
|
add_cloud_files: Add Cloud Files
|
|
94
|
+
errors:
|
|
95
|
+
missing_destination_context: This cloud file download is missing required destination information and can no longer be safely processed. Please re-select the file and try the import again.
|
|
94
96
|
csv:
|
|
95
97
|
add_csv_to_import: 'Add CSV or ZIP File to Import:'
|
|
96
98
|
add_files_hint: Choose files to upload. The filenames must be unique, and the filenames must be referenced in a column called 'file' in the accompanying CSV file.
|
data/lib/bulkrax/engine.rb
CHANGED
|
@@ -26,7 +26,7 @@ module Bulkrax
|
|
|
26
26
|
config.after_initialize do
|
|
27
27
|
# We want to ensure that Bulkrax is earlier in the lookup for view_paths than Hyrax. That is
|
|
28
28
|
# we favor view in Bulkrax over those in Hyrax.
|
|
29
|
-
if
|
|
29
|
+
if Bulkrax.hyrax_loaded?
|
|
30
30
|
my_engine_root = Bulkrax::Engine.root.to_s
|
|
31
31
|
hyrax_engine_root = Hyrax::Engine.root.to_s
|
|
32
32
|
paths = ActionController::Base.view_paths.collect(&:to_s)
|
data/lib/bulkrax/version.rb
CHANGED
data/lib/bulkrax.rb
CHANGED
|
@@ -6,6 +6,7 @@ require 'active_support/all'
|
|
|
6
6
|
|
|
7
7
|
require 'coderay'
|
|
8
8
|
require 'csv'
|
|
9
|
+
require 'deprecation'
|
|
9
10
|
require 'denormalize_fields'
|
|
10
11
|
require 'erb'
|
|
11
12
|
require 'iso8601'
|
|
@@ -13,10 +14,9 @@ require 'language_list'
|
|
|
13
14
|
require 'marcel'
|
|
14
15
|
require 'nokogiri'
|
|
15
16
|
require 'ostruct'
|
|
17
|
+
require 'ssrf_filter'
|
|
16
18
|
require 'zip'
|
|
17
19
|
|
|
18
|
-
require 'wings/custom_queries/find_by_source_identifier'
|
|
19
|
-
|
|
20
20
|
def conditional_require(gem_name)
|
|
21
21
|
require gem_name
|
|
22
22
|
rescue LoadError
|
|
@@ -31,6 +31,10 @@ module Bulkrax
|
|
|
31
31
|
extend self # rubocop:disable Style/ModuleFunction
|
|
32
32
|
extend Forwardable
|
|
33
33
|
|
|
34
|
+
def hyrax_loaded?
|
|
35
|
+
defined?(::Hyrax::Engine) ? true : false
|
|
36
|
+
end
|
|
37
|
+
|
|
34
38
|
##
|
|
35
39
|
# @api public
|
|
36
40
|
class Configuration
|
|
@@ -85,7 +89,7 @@ module Bulkrax
|
|
|
85
89
|
def solr_key_for_member_file_ids
|
|
86
90
|
return @solr_key_for_member_file_ids if @solr_key_for_member_file_ids.present?
|
|
87
91
|
|
|
88
|
-
return "member_ids_ssim" if
|
|
92
|
+
return "member_ids_ssim" if Bulkrax.hyrax_loaded?
|
|
89
93
|
|
|
90
94
|
"#{file_model_class.name.to_s.underscore}_ids_ssim"
|
|
91
95
|
end
|
|
@@ -110,10 +114,13 @@ module Bulkrax
|
|
|
110
114
|
end
|
|
111
115
|
|
|
112
116
|
def collection_model_class
|
|
113
|
-
@collection_model_class ||= Collection if
|
|
117
|
+
@collection_model_class ||= Collection if Bulkrax.hyrax_loaded?
|
|
114
118
|
end
|
|
115
119
|
|
|
116
|
-
|
|
120
|
+
def collection_model_class=(klass)
|
|
121
|
+
@collection_model_names = nil
|
|
122
|
+
@collection_model_class = klass
|
|
123
|
+
end
|
|
117
124
|
|
|
118
125
|
def collection_model_internal_resource
|
|
119
126
|
# WARN: Using #try on :internal_resource can yield unexpected results.
|
|
@@ -129,11 +136,30 @@ module Bulkrax
|
|
|
129
136
|
instance.respond_to?(:internal_resource) ? instance.internal_resource : collection_model_class.to_s
|
|
130
137
|
end
|
|
131
138
|
|
|
139
|
+
# A sheet written against either side of a Wings pair has to import, since
|
|
140
|
+
# both names resolve to the same object.
|
|
141
|
+
def collection_model_names
|
|
142
|
+
@collection_model_names ||=
|
|
143
|
+
model_names_for(collection_model_class, twin: curation_concern_twin_for(collection_model_class))
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
attr_writer :collection_model_names
|
|
147
|
+
|
|
148
|
+
def collection_model_name?(model_name)
|
|
149
|
+
name = model_name.to_s.strip
|
|
150
|
+
return false if name.empty?
|
|
151
|
+
|
|
152
|
+
collection_model_names.any? { |known| name.casecmp(known).zero? }
|
|
153
|
+
end
|
|
154
|
+
|
|
132
155
|
def file_model_class
|
|
133
|
-
@file_model_class ||=
|
|
156
|
+
@file_model_class ||= Bulkrax.hyrax_loaded? ? ::FileSet : File
|
|
134
157
|
end
|
|
135
158
|
|
|
136
|
-
|
|
159
|
+
def file_model_class=(klass)
|
|
160
|
+
@file_model_names = nil
|
|
161
|
+
@file_model_class = klass
|
|
162
|
+
end
|
|
137
163
|
|
|
138
164
|
def file_model_internal_resource
|
|
139
165
|
# WARN: Using #try on :internal_resource can yield unexpected results.
|
|
@@ -149,8 +175,43 @@ module Bulkrax
|
|
|
149
175
|
instance.respond_to?(:internal_resource) ? instance.internal_resource : file_model_class.to_s
|
|
150
176
|
end
|
|
151
177
|
|
|
178
|
+
# As #collection_model_names, for file set rows. Wings pairs file sets by
|
|
179
|
+
# namespace rather than a `Resource` suffix: `Wings::ModelRegistry` maps
|
|
180
|
+
# both `FileSet` and `Hyrax::FileSet` onto the same object, so whichever
|
|
181
|
+
# one is configured, the other is its twin.
|
|
182
|
+
def file_model_names
|
|
183
|
+
@file_model_names ||= model_names_for(file_model_class, twin: file_set_twin_for(file_model_class))
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
attr_writer :file_model_names
|
|
187
|
+
|
|
188
|
+
def file_model_name?(model_name)
|
|
189
|
+
name = model_name.to_s.strip
|
|
190
|
+
return false if name.empty?
|
|
191
|
+
|
|
192
|
+
file_model_names.any? { |known| name.casecmp(known).zero? }
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# Constantizing is what distinguishes a Wings install from a name that
|
|
196
|
+
# merely looks like one.
|
|
197
|
+
def model_names_for(klass, twin:)
|
|
198
|
+
names = [klass.to_s]
|
|
199
|
+
names << twin.to_s if twin.present? && twin.to_s.safe_constantize
|
|
200
|
+
names.uniq
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def curation_concern_twin_for(klass)
|
|
204
|
+
name = klass.to_s
|
|
205
|
+
name.end_with?('Resource') ? name.delete_suffix('Resource') : "#{name}Resource"
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def file_set_twin_for(klass)
|
|
209
|
+
name = klass.to_s
|
|
210
|
+
name.start_with?('Hyrax::') ? name.delete_prefix('Hyrax::') : "Hyrax::#{name}"
|
|
211
|
+
end
|
|
212
|
+
|
|
152
213
|
def curation_concerns
|
|
153
|
-
@curation_concerns ||=
|
|
214
|
+
@curation_concerns ||= Bulkrax.hyrax_loaded? ? ::Hyrax.config.curation_concerns : []
|
|
154
215
|
end
|
|
155
216
|
|
|
156
217
|
attr_writer :curation_concerns
|
|
@@ -176,7 +237,7 @@ module Bulkrax
|
|
|
176
237
|
# @return [String, Proc]
|
|
177
238
|
def ingest_queue_name
|
|
178
239
|
return @ingest_queue_name if @ingest_queue_name.present?
|
|
179
|
-
return Hyrax.config.ingest_queue_name if
|
|
240
|
+
return Hyrax.config.ingest_queue_name if Bulkrax.hyrax_loaded?
|
|
180
241
|
:import
|
|
181
242
|
end
|
|
182
243
|
|
|
@@ -224,6 +285,9 @@ module Bulkrax
|
|
|
224
285
|
:collection_model_class,
|
|
225
286
|
:collection_model_internal_resource,
|
|
226
287
|
:collection_model_class=,
|
|
288
|
+
:collection_model_name?,
|
|
289
|
+
:collection_model_names,
|
|
290
|
+
:collection_model_names=,
|
|
227
291
|
:curation_concerns,
|
|
228
292
|
:curation_concerns=,
|
|
229
293
|
:curation_concern_internal_resources,
|
|
@@ -240,6 +304,9 @@ module Bulkrax
|
|
|
240
304
|
:file_model_class,
|
|
241
305
|
:file_model_class=,
|
|
242
306
|
:file_model_internal_resource,
|
|
307
|
+
:file_model_name?,
|
|
308
|
+
:file_model_names,
|
|
309
|
+
:file_model_names=,
|
|
243
310
|
:fill_in_blank_source_identifiers,
|
|
244
311
|
:fill_in_blank_source_identifiers=,
|
|
245
312
|
:generated_metadata_mapping,
|
|
@@ -433,7 +500,7 @@ module Bulkrax
|
|
|
433
500
|
end
|
|
434
501
|
|
|
435
502
|
def fallback_user_for_importer_exporter_processing
|
|
436
|
-
return User.batch_user if
|
|
503
|
+
return User.batch_user if Bulkrax.hyrax_loaded? && User.respond_to?(:batch_user)
|
|
437
504
|
|
|
438
505
|
raise "We have no fallback user available for Bulkrax.fallback_user_for_importer_exporter_processing"
|
|
439
506
|
end
|