bulkrax 9.5.2 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +49 -0
- data/app/controllers/bulkrax/application_controller.rb +83 -0
- data/app/controllers/bulkrax/entries_controller.rb +16 -13
- data/app/controllers/bulkrax/exporters_controller.rb +21 -25
- data/app/controllers/bulkrax/guided_imports_controller.rb +4 -8
- data/app/controllers/bulkrax/importers_controller.rb +43 -87
- data/app/controllers/concerns/bulkrax/datatables_behavior.rb +18 -12
- data/app/controllers/concerns/bulkrax/importer_file_handler.rb +9 -55
- data/app/factories/bulkrax/object_factory.rb +2 -2
- data/app/factories/bulkrax/valkyrie_object_factory.rb +29 -17
- data/app/helpers/bulkrax/validation_helper.rb +1 -1
- data/app/jobs/bulkrax/create_relationships_job.rb +1 -1
- data/app/jobs/bulkrax/download_cloud_file_job.rb +21 -156
- data/app/jobs/bulkrax/import_collection_job.rb +9 -2
- data/app/matchers/bulkrax/application_matcher.rb +5 -2
- data/app/models/bulkrax/csv_entry.rb +1 -1
- data/app/models/bulkrax/current.rb +7 -0
- data/app/models/bulkrax/export_scope.rb +21 -0
- data/app/models/bulkrax/exporter.rb +2 -2
- data/app/models/bulkrax/importer.rb +1 -1
- data/app/models/concerns/bulkrax/ability_behavior.rb +139 -0
- data/app/models/concerns/bulkrax/has_matchers.rb +1 -1
- data/app/models/concerns/bulkrax/import_behavior.rb +3 -3
- data/app/parsers/bulkrax/application_parser.rb +10 -10
- data/app/parsers/bulkrax/bagit_parser.rb +3 -10
- data/app/parsers/bulkrax/csv_parser.rb +28 -58
- data/app/parsers/bulkrax/oai_dc_parser.rb +1 -1
- data/app/parsers/bulkrax/parser_export_record_set.rb +26 -8
- data/app/parsers/concerns/bulkrax/csv_parser/csv_template_generation.rb +1 -1
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation.rb +1 -2
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_helpers.rb +18 -1
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_hierarchy.rb +2 -2
- data/app/search_builders/bulkrax/export_search_builder.rb +9 -0
- data/app/services/bulkrax/csv_template/column_builder.rb +2 -2
- data/app/services/bulkrax/csv_template/mapping_manager.rb +9 -7
- data/app/services/bulkrax/csv_template/schema_analyzer.rb +1 -5
- data/config/locales/bulkrax.en.yml +0 -2
- data/lib/bulkrax/engine.rb +1 -1
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +76 -11
- data/lib/generators/bulkrax/install_generator.rb +22 -10
- data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb +1 -1
- metadata +32 -34
- data/app/errors/bulkrax/unsafe_cloud_source_error.rb +0 -11
- data/app/errors/bulkrax/unsafe_path_error.rb +0 -8
- data/app/parsers/bulkrax/application_parser.rb.orig +0 -545
- data/app/services/bulkrax/cloud_source_validator.rb +0 -123
- data/app/services/bulkrax/safe_path.rb +0 -129
- data/lib/wings/custom_queries/find_by_source_identifier.rb +0 -41
|
@@ -1,166 +1,31 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
-
|
|
3
2
|
module Bulkrax
|
|
4
3
|
class DownloadCloudFileJob < ApplicationJob
|
|
5
4
|
queue_as Bulkrax.config.ingest_queue_name
|
|
6
5
|
|
|
7
|
-
|
|
8
|
-
#
|
|
9
|
-
# `importer_id:` and `relative_filename:` are required keywords: the
|
|
10
|
-
# destination is always recomputed here, confined to this importer's
|
|
11
|
-
# own workspace, rather than trusted from a serialized job argument.
|
|
12
|
-
# `target_file` is a deprecated positional placeholder, kept only so
|
|
13
|
-
# that a pre-upgrade call shape (`file`, `target_file`) still binds
|
|
14
|
-
# without Ruby raising its own ArgumentError — it is never read, even
|
|
15
|
-
# when supplied alongside otherwise-valid keywords, so it can't be
|
|
16
|
-
# used to redirect the destination. A call made this way is missing
|
|
17
|
-
# the required keywords, so it raises below rather than running.
|
|
18
|
-
#
|
|
19
|
-
# Note: if using the file system provider, the mounted directory in
|
|
20
|
-
# browse_everything MUST be shared by web and worker servers
|
|
21
|
-
def perform(file, target_file = nil, importer_id: nil, relative_filename: nil)
|
|
22
|
-
raise Bulkrax::UnsafePathError, I18n.t('bulkrax.importer.browse_everything.errors.missing_destination_context') if importer_id.blank? || relative_filename.blank?
|
|
23
|
-
|
|
24
|
-
# `file` and `relative_filename` are independent arguments and may
|
|
25
|
-
# not agree (a tampered or replayed job call could supply either
|
|
26
|
-
# one separately) — check the raw file_name directly rather than
|
|
27
|
-
# assuming it was already validated by whatever enqueued this job.
|
|
28
|
-
Bulkrax::SafePath.reject_unsafe_entry!(file['file_name'])
|
|
29
|
-
|
|
30
|
-
importer = Bulkrax::Importer.find(importer_id)
|
|
31
|
-
target_file = Bulkrax::SafePath.confine!(importer.parser.path_for_import, relative_filename, require_file: true)
|
|
32
|
-
|
|
33
|
-
# Re-validate the source here too: this job may run on a different
|
|
34
|
-
# process/host than the one that enqueued it, and could be invoked
|
|
35
|
-
# directly with a tampered or stale `file` argument.
|
|
36
|
-
Bulkrax::CloudSourceValidator.validate!(file)
|
|
37
|
-
|
|
38
|
-
download(file, target_file)
|
|
39
|
-
Rails.logger.info "Bulkrax: cloud file download complete for #{file['file_name']}"
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
private
|
|
43
|
-
|
|
44
|
-
# Downloads through a temp file in the *same* directory as
|
|
45
|
-
# `target_file` (so the final move is same-filesystem and atomic),
|
|
46
|
-
# named with a random suffix rather than anything derived from
|
|
47
|
-
# caller-controlled input, and removed on any failure. This means a
|
|
48
|
-
# failed or non-success download can never truncate or corrupt an
|
|
49
|
-
# already-good destination file from a previous successful import.
|
|
50
|
-
def download(file, target_file)
|
|
51
|
-
temp_file = "#{target_file}.#{SecureRandom.hex(8)}.part"
|
|
52
|
-
uri = Addressable::URI.parse(file['url'].to_s)
|
|
53
|
-
if uri.scheme == 'file'
|
|
54
|
-
download_file_source(uri, temp_file)
|
|
55
|
-
else
|
|
56
|
-
download_http_source(file, URI.parse(file['url'].to_s), temp_file)
|
|
57
|
-
end
|
|
58
|
-
FileUtils.mv(temp_file, target_file)
|
|
59
|
-
ensure
|
|
60
|
-
FileUtils.rm_f(temp_file) if temp_file && File.exist?(temp_file)
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
# The source was already confined to the configured filesystem
|
|
64
|
-
# provider's root by {Bulkrax::CloudSourceValidator}.
|
|
65
|
-
def download_file_source(uri, temp_file)
|
|
66
|
-
source_path = URI::DEFAULT_PARSER.unescape(uri.path.to_s)
|
|
67
|
-
IO.copy_stream(source_path, temp_file)
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
# Resolved, IP-pinned, redirect-free, and streamed straight to disk in
|
|
71
|
-
# bounded-size chunks — rather than handing the URL back to a transport
|
|
72
|
-
# (browse-everything's Retriever) that re-resolves the hostname with no
|
|
73
|
-
# SSRF protection at all. This is a *separate* connection from the one
|
|
74
|
-
# {Bulkrax::CloudSourceValidator}'s HEAD check made; each independently
|
|
75
|
-
# applies the same validation at its own connection time, so there is
|
|
76
|
-
# no single "pinned connection" shared between validation and download.
|
|
77
|
-
#
|
|
78
|
-
# ssrf_filter 1.0.8's public `get`/`head` API has no response-streaming
|
|
79
|
-
# hook: `fetch_once` calls `Net::HTTP#request` with no block, which
|
|
80
|
-
# reads the entire response body into memory before returning; the
|
|
81
|
-
# block `get`/`head` accept is for customizing the *outgoing* request,
|
|
82
|
-
# not consuming the response. So we can't use `SsrfFilter.get` here
|
|
83
|
-
# without buffering the whole remote file in memory first.
|
|
84
|
-
#
|
|
85
|
-
# Reuse ssrf_filter's resolver and IP policy, then pin the validated
|
|
86
|
-
# address with Net::HTTP#ipaddr=. Keeping the original hostname on
|
|
87
|
-
# the HTTP connection preserves native TLS SNI and certificate
|
|
88
|
-
# verification without relying on ssrf_filter's TLS monkeypatches.
|
|
89
|
-
def download_http_source(file, uri, temp_file)
|
|
90
|
-
headers = Bulkrax::CloudSourceValidator.request_headers(file)
|
|
91
|
-
stream_download(uri, headers) { |response| handle_response(file, response, temp_file) }
|
|
92
|
-
rescue SsrfFilter::Error
|
|
93
|
-
# The validator's own HEAD check already ran, but DNS can change
|
|
94
|
-
# between that check and this GET — same rationale as
|
|
95
|
-
# Bulkrax::CloudSourceValidator#validate_http_scheme!, and the same
|
|
96
|
-
# static, class-agnostic message for the same reason.
|
|
97
|
-
raise Bulkrax::UnsafeCloudSourceError, 'blocked unsafe remote source'
|
|
98
|
-
end
|
|
6
|
+
include ActionView::Helpers::NumberHelper
|
|
99
7
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
# as the `Host` header and used for TLS SNI/certificate verification
|
|
120
|
-
# instead of the IP. Yields the live `Net::HTTPResponse` to the block
|
|
121
|
-
# *before* its body is read, so the caller can stream it.
|
|
122
|
-
def stream_download(uri, headers)
|
|
123
|
-
raise SsrfFilter::InvalidUriScheme, "URI scheme '#{uri.scheme}' not in whitelist" unless
|
|
124
|
-
SsrfFilter::DEFAULT_SCHEME_WHITELIST.include?(uri.scheme)
|
|
125
|
-
|
|
126
|
-
hostname = uri.hostname
|
|
127
|
-
ip_addresses = SsrfFilter::DEFAULT_RESOLVER.call(hostname)
|
|
128
|
-
raise SsrfFilter::UnresolvedHostname, "Could not resolve hostname '#{hostname}'" if ip_addresses.empty?
|
|
129
|
-
|
|
130
|
-
public_addresses = ip_addresses.reject { |ip| unsafe_ip_address?(ip) }
|
|
131
|
-
raise SsrfFilter::PrivateIPAddress, "Hostname '#{hostname}' has no public ip addresses" if public_addresses.empty?
|
|
132
|
-
|
|
133
|
-
request = Net::HTTP::Get.new(uri)
|
|
134
|
-
request['host'] = host_header(hostname, uri)
|
|
135
|
-
headers.each { |key, value| request[key] = value }
|
|
136
|
-
|
|
137
|
-
http = Net::HTTP.new(hostname, uri.port, nil)
|
|
138
|
-
http.ipaddr = public_addresses.sample.to_s
|
|
139
|
-
http.use_ssl = (uri.scheme == 'https')
|
|
140
|
-
http.start { |connection| connection.request(request) { |response| yield response } }
|
|
141
|
-
end
|
|
142
|
-
|
|
143
|
-
# Mirrors `SsrfFilter.unsafe_ip_address?`, but against its public
|
|
144
|
-
# `IPV4_BLACKLIST`/`IPV6_BLACKLIST` constants directly rather than
|
|
145
|
-
# calling that private method — we only ever check single resolved
|
|
146
|
-
# addresses here (never a CIDR range), so the masked-range guard it
|
|
147
|
-
# also carries doesn't apply.
|
|
148
|
-
def unsafe_ip_address?(ip_address)
|
|
149
|
-
return SsrfFilter::IPV4_BLACKLIST.any? { |range| range.include?(ip_address) } if ip_address.ipv4?
|
|
150
|
-
return SsrfFilter::IPV6_BLACKLIST.any? { |range| range.include?(ip_address) } if ip_address.ipv6?
|
|
151
|
-
|
|
152
|
-
true
|
|
153
|
-
end
|
|
154
|
-
|
|
155
|
-
# Matches `SsrfFilter.host_header`: the `Host` header should carry the
|
|
156
|
-
# real hostname (not the pinned IP we actually connect to), with the
|
|
157
|
-
# port suffixed only when it isn't the scheme's default.
|
|
158
|
-
def host_header(hostname, uri)
|
|
159
|
-
if (uri.port == 80 && uri.scheme == 'http') || (uri.port == 443 && uri.scheme == 'https')
|
|
160
|
-
hostname
|
|
161
|
-
else
|
|
162
|
-
"#{hostname}:#{uri.port}"
|
|
8
|
+
# Retrieve cloud file and write to the imports directory
|
|
9
|
+
# Note: if using the file system, the mounted directory in
|
|
10
|
+
# browse_everything MUST be shared by web and worker servers
|
|
11
|
+
def perform(file, target_file)
|
|
12
|
+
retriever = BrowseEverything::Retriever.new
|
|
13
|
+
last_logged_time = Time.zone.now
|
|
14
|
+
log_interval = 3.seconds
|
|
15
|
+
|
|
16
|
+
retriever.download(file, target_file) do |filename, retrieved, total|
|
|
17
|
+
percentage = (retrieved.to_f / total.to_f) * 100
|
|
18
|
+
current_time = Time.zone.now
|
|
19
|
+
|
|
20
|
+
if (current_time - last_logged_time) >= log_interval
|
|
21
|
+
# Use number_to_human_size for formatting
|
|
22
|
+
readable_retrieved = number_to_human_size(retrieved)
|
|
23
|
+
readable_total = number_to_human_size(total)
|
|
24
|
+
Rails.logger.info "Downloaded #{readable_retrieved} of #{readable_total}, #{filename}: #{percentage.round}% complete"
|
|
25
|
+
last_logged_time = current_time
|
|
26
|
+
end
|
|
163
27
|
end
|
|
28
|
+
Rails.logger.info "Download complete: #{file['url']} to #{target_file}"
|
|
164
29
|
end
|
|
165
30
|
end
|
|
166
31
|
end
|
|
@@ -10,8 +10,15 @@ module Bulkrax
|
|
|
10
10
|
begin
|
|
11
11
|
entry.build
|
|
12
12
|
entry.save!
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
# An entry that fails inside #build records the error on itself rather
|
|
14
|
+
# than raising, so its status is what says whether this run succeeded.
|
|
15
|
+
if entry.succeeded?
|
|
16
|
+
ImporterRun.increment_counter(:processed_records, args[1])
|
|
17
|
+
ImporterRun.increment_counter(:processed_collections, args[1])
|
|
18
|
+
else
|
|
19
|
+
ImporterRun.increment_counter(:failed_records, args[1])
|
|
20
|
+
ImporterRun.increment_counter(:failed_collections, args[1])
|
|
21
|
+
end
|
|
15
22
|
ImporterRun.decrement_counter(:enqueued_records, args[1]) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
|
|
16
23
|
rescue => e
|
|
17
24
|
ImporterRun.increment_counter(:failed_records, args[1])
|
|
@@ -67,10 +67,13 @@ module Bulkrax
|
|
|
67
67
|
end
|
|
68
68
|
|
|
69
69
|
def parse_subject(src)
|
|
70
|
-
string = src.strip
|
|
70
|
+
string = src.strip
|
|
71
71
|
return if string.blank?
|
|
72
72
|
|
|
73
|
-
string.
|
|
73
|
+
return string.sub(%r{\Ahttps?}i, &:downcase) if string.match?(%r{\Ahttps?://}i)
|
|
74
|
+
|
|
75
|
+
downcased = string.downcase
|
|
76
|
+
downcased.slice(0, 1).capitalize + downcased.slice(1..-1)
|
|
74
77
|
end
|
|
75
78
|
|
|
76
79
|
def parse_types(src)
|
|
@@ -141,7 +141,7 @@ module Bulkrax
|
|
|
141
141
|
|
|
142
142
|
def add_metadata_for_model
|
|
143
143
|
if factory_class.present? && factory_class == Bulkrax.collection_model_class
|
|
144
|
-
add_collection_type_gid if
|
|
144
|
+
add_collection_type_gid if Bulkrax.hyrax_loaded?
|
|
145
145
|
# add any additional collection metadata methods here
|
|
146
146
|
elsif factory_class == Bulkrax.file_model_class
|
|
147
147
|
validate_presence_of_filename!
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
# A minimal scope adapter that satisfies the Blacklight::SearchBuilder and
|
|
5
|
+
# Hydra::AccessControlsEnforcement interface for export permission filtering.
|
|
6
|
+
#
|
|
7
|
+
# Blacklight::SearchBuilder expects the scope to respond to +blacklight_config+
|
|
8
|
+
# and +current_ability+. In the Hyrax catalog this is the controller; for
|
|
9
|
+
# Bulkrax exports a lightweight PORO is sufficient.
|
|
10
|
+
class ExportScope
|
|
11
|
+
attr_reader :current_ability
|
|
12
|
+
|
|
13
|
+
def initialize(ability)
|
|
14
|
+
@current_ability = ability
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def blacklight_config
|
|
18
|
+
@blacklight_config ||= Blacklight::Configuration.new
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -69,7 +69,7 @@ module Bulkrax
|
|
|
69
69
|
end
|
|
70
70
|
|
|
71
71
|
def workflow_status_list
|
|
72
|
-
Sipity::WorkflowState.all.map { |s| [s.name&.titleize, s.name] }.uniq if
|
|
72
|
+
Sipity::WorkflowState.all.map { |s| [s.name&.titleize, s.name] }.uniq if Bulkrax.hyrax_loaded?
|
|
73
73
|
end
|
|
74
74
|
|
|
75
75
|
# If field_mapping is empty, setup a default based on the export_properties
|
|
@@ -84,7 +84,7 @@ module Bulkrax
|
|
|
84
84
|
end
|
|
85
85
|
|
|
86
86
|
def export_from_list
|
|
87
|
-
if
|
|
87
|
+
if Bulkrax.hyrax_loaded?
|
|
88
88
|
[
|
|
89
89
|
[I18n.t('bulkrax.exporter.labels.importer'), 'importer'],
|
|
90
90
|
[I18n.t('bulkrax.exporter.labels.collection'), 'collection'],
|
|
@@ -18,7 +18,7 @@ module Bulkrax
|
|
|
18
18
|
has_many :entries, as: :importerexporter, dependent: :destroy
|
|
19
19
|
|
|
20
20
|
validates :name, presence: true
|
|
21
|
-
validates :admin_set_id, presence: true if
|
|
21
|
+
validates :admin_set_id, presence: true if Bulkrax.hyrax_loaded?
|
|
22
22
|
validates :parser_klass, presence: true
|
|
23
23
|
|
|
24
24
|
delegate :create_parent_child_relationships, :valid_import?, :write_errored_entries_file, :visibility, to: :parser
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
# Concern providing Bulkrax-specific ability methods.
|
|
5
|
+
#
|
|
6
|
+
# Include this in your application's Ability class to gain default
|
|
7
|
+
# implementations of all Bulkrax authorization methods. Override any
|
|
8
|
+
# method to customize authorization behavior for your application.
|
|
9
|
+
#
|
|
10
|
+
# ## Wiring CanCan rules
|
|
11
|
+
#
|
|
12
|
+
# Call +bulkrax_default_abilities+ from your Ability's +initialize+ (or add
|
|
13
|
+
# it to +self.ability_logic+) to register CanCan +can+ rules for all
|
|
14
|
+
# Bulkrax resources:
|
|
15
|
+
#
|
|
16
|
+
# class Ability
|
|
17
|
+
# include Hydra::Ability
|
|
18
|
+
# include Bulkrax::AbilityBehavior
|
|
19
|
+
# self.ability_logic += [:bulkrax_default_abilities]
|
|
20
|
+
# end
|
|
21
|
+
#
|
|
22
|
+
# The method uses the four predicate hooks below as guards, so you can
|
|
23
|
+
# control who gets which rules simply by overriding those predicates.
|
|
24
|
+
#
|
|
25
|
+
# Example:
|
|
26
|
+
#
|
|
27
|
+
# class Ability
|
|
28
|
+
# include Hydra::Ability
|
|
29
|
+
# include Bulkrax::AbilityBehavior
|
|
30
|
+
# self.ability_logic += [:bulkrax_default_abilities]
|
|
31
|
+
#
|
|
32
|
+
# # Grant import/export access to users who can create works
|
|
33
|
+
# def can_import_works?
|
|
34
|
+
# can_create_any_work?
|
|
35
|
+
# end
|
|
36
|
+
#
|
|
37
|
+
# def can_export_works?
|
|
38
|
+
# can_create_any_work?
|
|
39
|
+
# end
|
|
40
|
+
#
|
|
41
|
+
# # Grant admin-level importer access to site admins
|
|
42
|
+
# def can_admin_importers?
|
|
43
|
+
# current_user.admin?
|
|
44
|
+
# end
|
|
45
|
+
# end
|
|
46
|
+
module AbilityBehavior
|
|
47
|
+
extend ActiveSupport::Concern
|
|
48
|
+
|
|
49
|
+
# Returns true if the current user may use importer functionality at all.
|
|
50
|
+
# Override in your Ability class to customize.
|
|
51
|
+
def can_import_works?
|
|
52
|
+
false
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Returns true if the current user may use exporter functionality at all.
|
|
56
|
+
# Override in your Ability class to customize.
|
|
57
|
+
def can_export_works?
|
|
58
|
+
false
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Returns true if the current user may administer ALL importers —
|
|
62
|
+
# i.e., view, edit, update, or destroy any importer regardless of ownership.
|
|
63
|
+
# Defaults to false; override to grant admin access.
|
|
64
|
+
def can_admin_importers?
|
|
65
|
+
false
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Returns true if the current user may administer ALL exporters —
|
|
69
|
+
# i.e., view, edit, update, or destroy any exporter regardless of ownership.
|
|
70
|
+
# Defaults to false; override to grant admin access.
|
|
71
|
+
def can_admin_exporters?
|
|
72
|
+
false
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Registers CanCan +can+ rules for all Bulkrax resources.
|
|
76
|
+
#
|
|
77
|
+
# Call this from your Ability's +initialize+ or add it to
|
|
78
|
+
# +self.ability_logic+. The predicate hooks above control which rules are
|
|
79
|
+
# granted; override them to tailor access.
|
|
80
|
+
#
|
|
81
|
+
# Rules for Importer and Exporter use hash-form conditions (SQL-generatable)
|
|
82
|
+
# so they work with +Model.accessible_by(current_ability)+. Rules for Entry
|
|
83
|
+
# use block-form conditions (checked in Ruby) because authorization must
|
|
84
|
+
# traverse the parent association — do not call +Entry.accessible_by()+;
|
|
85
|
+
# scope entries through the parent instead (e.g. +@importer.entries+).
|
|
86
|
+
def bulkrax_default_abilities
|
|
87
|
+
# Skip all rules when there is no authenticated user; avoids nil-id bugs
|
|
88
|
+
# and prevents block rules from matching NULL user_id records.
|
|
89
|
+
return unless current_user&.id
|
|
90
|
+
|
|
91
|
+
# Map Bulkrax-specific non-CRUD actions to their closest CanCan equivalents
|
|
92
|
+
# so that load_and_authorize_resource can authorize them without requiring
|
|
93
|
+
# a dedicated `can` declaration for each custom action name.
|
|
94
|
+
alias_action :entry_table, :importer_table, :exporter_table,
|
|
95
|
+
:original_file, :export_errors, :upload_corrected_entries, to: :read
|
|
96
|
+
alias_action :continue, :upload_corrected_entries_file, to: :update
|
|
97
|
+
|
|
98
|
+
grant_importer_abilities
|
|
99
|
+
grant_exporter_abilities
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def grant_importer_abilities
|
|
103
|
+
if can_admin_importers?
|
|
104
|
+
# Prefer a single unconditional rule for admins. Older CanCanCan/
|
|
105
|
+
# ActiveRecord combinations used by the Hyrax 4 appraisal can fail when
|
|
106
|
+
# accessible_by has to merge unconditional and hash-conditioned rules.
|
|
107
|
+
can :manage, Bulkrax::Importer
|
|
108
|
+
can [:read, :update, :destroy], Bulkrax::Entry do |entry|
|
|
109
|
+
entry.importerexporter.is_a?(Bulkrax::Importer)
|
|
110
|
+
end
|
|
111
|
+
elsif can_import_works?
|
|
112
|
+
can :create, Bulkrax::Importer
|
|
113
|
+
can [:read, :update, :destroy], Bulkrax::Importer, user_id: current_user.id
|
|
114
|
+
can [:read, :update, :destroy], Bulkrax::Entry do |entry|
|
|
115
|
+
entry.importerexporter.is_a?(Bulkrax::Importer) &&
|
|
116
|
+
entry.importerexporter.user_id == current_user.id
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def grant_exporter_abilities
|
|
122
|
+
if can_admin_exporters?
|
|
123
|
+
# Prefer a single unconditional rule for admins. See importer rule
|
|
124
|
+
# above for the Hyrax 4 accessible_by compatibility note.
|
|
125
|
+
can :manage, Bulkrax::Exporter
|
|
126
|
+
can [:read, :update, :destroy], Bulkrax::Entry do |entry|
|
|
127
|
+
entry.importerexporter.is_a?(Bulkrax::Exporter)
|
|
128
|
+
end
|
|
129
|
+
elsif can_export_works?
|
|
130
|
+
can :create, Bulkrax::Exporter
|
|
131
|
+
can [:read, :update, :destroy], Bulkrax::Exporter, user_id: current_user.id
|
|
132
|
+
can [:read, :update, :destroy], Bulkrax::Entry do |entry|
|
|
133
|
+
entry.importerexporter.is_a?(Bulkrax::Exporter) &&
|
|
134
|
+
entry.importerexporter.user_id == current_user.id
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
@@ -230,7 +230,7 @@ module Bulkrax
|
|
|
230
230
|
# @return [Array] hyrax fields
|
|
231
231
|
def field_to(field)
|
|
232
232
|
fields = mapping&.map do |key, value|
|
|
233
|
-
|
|
233
|
+
next unless value
|
|
234
234
|
|
|
235
235
|
if value['from'].instance_of?(Array)
|
|
236
236
|
key if value['from'].include?(field) || key == field
|
|
@@ -34,7 +34,7 @@ module Bulkrax
|
|
|
34
34
|
# TODO: This guard clause is not necessary as we can handle it in the
|
|
35
35
|
# underlying factory. However, to do that requires adjusting about 7
|
|
36
36
|
# failing specs. So for now this refactor appears acceptable
|
|
37
|
-
return unless
|
|
37
|
+
return unless Bulkrax.hyrax_loaded?
|
|
38
38
|
return unless self.class.to_s.include?("Collection")
|
|
39
39
|
factory.add_user_to_collection_permissions(collection: @item, user: user)
|
|
40
40
|
end
|
|
@@ -87,7 +87,7 @@ module Bulkrax
|
|
|
87
87
|
end
|
|
88
88
|
|
|
89
89
|
def add_admin_set_id
|
|
90
|
-
return unless
|
|
90
|
+
return unless Bulkrax.hyrax_loaded?
|
|
91
91
|
|
|
92
92
|
self.parsed_metadata['admin_set_id'] = importerexporter.admin_set_id if self.parsed_metadata['admin_set_id'].blank?
|
|
93
93
|
end
|
|
@@ -161,7 +161,7 @@ module Bulkrax
|
|
|
161
161
|
# @param field [String] name of the controlled property
|
|
162
162
|
# @return [Boolean] provided value is a present, active authority ID for the provided field
|
|
163
163
|
def active_id_for_authority?(value, field)
|
|
164
|
-
return false unless
|
|
164
|
+
return false unless Bulkrax.hyrax_loaded?
|
|
165
165
|
field_service = ('Hyrax::' + "#{field}_service".camelcase).constantize
|
|
166
166
|
active_authority_ids = field_service.new.active_elements.map { |ae| ae['id'] }
|
|
167
167
|
|
|
@@ -472,27 +472,27 @@ module Bulkrax
|
|
|
472
472
|
# Zip Slip preflight — reject entries whose names are obviously unsafe
|
|
473
473
|
# (absolute paths, `..` segments) before we touch the filesystem.
|
|
474
474
|
# {#safe_extract_path} is the final line of defense; this check just
|
|
475
|
-
# fails fast with a clear message.
|
|
476
|
-
# cloud-file destinations and sources share the same rules.
|
|
475
|
+
# fails fast with a clear message.
|
|
477
476
|
#
|
|
478
477
|
# @raise [Bulkrax::UnzipError] if the entry name is unsafe
|
|
479
478
|
def reject_unsafe_entry!(name)
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: name.dup.force_encoding(Encoding::UTF_8).scrub)
|
|
479
|
+
return unless name.start_with?('/') || name.split('/').include?('..')
|
|
480
|
+
raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: name)
|
|
483
481
|
end
|
|
484
482
|
|
|
485
483
|
# Zip Slip chokepoint. Resolves `relative_dest` against `dest_dir` and
|
|
486
484
|
# returns the absolute destination path — but only if it stays inside
|
|
487
|
-
# `dest_dir
|
|
488
|
-
#
|
|
489
|
-
#
|
|
485
|
+
# `dest_dir`. Callers must use this value rather than building their
|
|
486
|
+
# own path with `File.join`, so the path returned is always safe by
|
|
487
|
+
# construction.
|
|
490
488
|
#
|
|
491
489
|
# @return [String] absolute destination path, validated to be inside `dest_dir`
|
|
492
490
|
# @raise [Bulkrax::UnzipError] if `relative_dest` escapes `dest_dir`
|
|
493
491
|
def safe_extract_path(dest_dir, relative_dest)
|
|
494
|
-
|
|
495
|
-
|
|
492
|
+
expanded_dest_dir = File.expand_path(dest_dir)
|
|
493
|
+
dest_path = File.expand_path(relative_dest.to_s, expanded_dest_dir)
|
|
494
|
+
return dest_path if dest_path == expanded_dest_dir
|
|
495
|
+
return dest_path if dest_path.start_with?("#{expanded_dest_dir}#{File::SEPARATOR}")
|
|
496
496
|
raise Bulkrax::UnzipError, I18n.t('bulkrax.importer.unzip.errors.unsafe_entry', name: relative_dest)
|
|
497
497
|
end
|
|
498
498
|
|
|
@@ -178,19 +178,12 @@ unless ENV.fetch('BULKRAX_NO_BAGIT', 'false').to_s == 'true'
|
|
|
178
178
|
# @todo - investigate getting directory structure
|
|
179
179
|
# @todo - investigate using perform_later, and having the importer check for
|
|
180
180
|
# DownloadCloudFileJob before it starts
|
|
181
|
-
def retrieve_cloud_files(files,
|
|
181
|
+
def retrieve_cloud_files(files, _importer)
|
|
182
182
|
# There should only be one zip file for Bagit, take the first
|
|
183
183
|
return if files['0'].blank?
|
|
184
|
-
|
|
185
|
-
file = files['0']
|
|
186
|
-
# Checked on the raw file_name before any normalization — see
|
|
187
|
-
# Bulkrax::CsvParser#plan_cloud_file for why this must come first.
|
|
188
|
-
Bulkrax::CloudSourceValidator.preflight!(file)
|
|
189
|
-
|
|
190
|
-
relative_filename = file['file_name'].tr(' ', '_')
|
|
191
|
-
target_file = Bulkrax::SafePath.confine!(path_for_import, relative_filename, require_file: true)
|
|
184
|
+
target_file = File.join(path_for_import, files['0']['file_name'].tr(' ', '_'))
|
|
192
185
|
# Now because we want the files in place before the importer runs
|
|
193
|
-
Bulkrax::DownloadCloudFileJob.perform_now(
|
|
186
|
+
Bulkrax::DownloadCloudFileJob.perform_now(files['0'], target_file)
|
|
194
187
|
return target_file
|
|
195
188
|
end
|
|
196
189
|
|
|
@@ -37,7 +37,6 @@ module Bulkrax
|
|
|
37
37
|
@records
|
|
38
38
|
end
|
|
39
39
|
|
|
40
|
-
# rubocop:disable Metrics/AbcSize
|
|
41
40
|
def build_records
|
|
42
41
|
@collections = []
|
|
43
42
|
@works = []
|
|
@@ -49,11 +48,9 @@ module Bulkrax
|
|
|
49
48
|
next unless r.key?(model_mapping)
|
|
50
49
|
|
|
51
50
|
model = r[model_mapping].nil? ? "" : r[model_mapping].strip
|
|
52
|
-
|
|
53
|
-
# We aren't right now because so many Bulkrax users are in between Fedora and Valkyrie
|
|
54
|
-
if model.casecmp('collection').zero? || model.casecmp('collectionresource').zero?
|
|
51
|
+
if Bulkrax.collection_model_name?(model)
|
|
55
52
|
@collections << r
|
|
56
|
-
elsif
|
|
53
|
+
elsif Bulkrax.file_model_name?(model)
|
|
57
54
|
@file_sets << r
|
|
58
55
|
else
|
|
59
56
|
@works << r
|
|
@@ -69,7 +66,6 @@ module Bulkrax
|
|
|
69
66
|
|
|
70
67
|
true
|
|
71
68
|
end
|
|
72
|
-
# rubocop:enabled Metrics/AbcSize
|
|
73
69
|
|
|
74
70
|
def collections
|
|
75
71
|
build_records if @collections.nil?
|
|
@@ -209,55 +205,24 @@ module Bulkrax
|
|
|
209
205
|
# @todo - investigate using perform_later, and having the importer check for
|
|
210
206
|
# DownloadCloudFileJob before it starts
|
|
211
207
|
def retrieve_cloud_files(files, importer)
|
|
212
|
-
plan = plan_cloud_files(files)
|
|
213
|
-
dispatch_cloud_files!(plan, importer)
|
|
214
|
-
importer[:parser_fields]['original_file_paths'] = plan.map { |_file, _relative_filename, target_file| target_file }
|
|
215
|
-
return nil
|
|
216
|
-
end
|
|
217
|
-
|
|
218
|
-
# Validates and resolves the whole batch, without enqueueing anything
|
|
219
|
-
# yet. Otherwise an earlier, valid file's job would already be running
|
|
220
|
-
# by the time a later file in the same batch is found to be unsafe.
|
|
221
|
-
# Split out from {#retrieve_cloud_files} so a caller (the importers
|
|
222
|
-
# controller) can persist the resolved paths before dispatching.
|
|
223
|
-
#
|
|
224
|
-
# @return [Array<Array(Hash, String, String)>] one entry per file: the
|
|
225
|
-
# file spec, its relative destination filename, and the resolved
|
|
226
|
-
# absolute target path
|
|
227
|
-
def plan_cloud_files(files)
|
|
228
208
|
files_path = File.join(path_for_import, 'files')
|
|
229
209
|
FileUtils.mkdir_p(files_path) unless File.exist?(files_path)
|
|
230
|
-
|
|
231
|
-
files.
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
# @return [Array(Hash, String, String)] the file spec, its relative
|
|
244
|
-
# destination filename, and the resolved absolute target path
|
|
245
|
-
def plan_cloud_file(file)
|
|
246
|
-
# fixes bug where auth headers do not get attached properly
|
|
247
|
-
if file['auth_header'].present?
|
|
248
|
-
file['headers'] ||= {}
|
|
249
|
-
file['headers'].merge!(file['auth_header'])
|
|
210
|
+
target_files = []
|
|
211
|
+
files.each_pair do |_key, file|
|
|
212
|
+
# fixes bug where auth headers do not get attached properly
|
|
213
|
+
if file['auth_header'].present?
|
|
214
|
+
file['headers'] ||= {}
|
|
215
|
+
file['headers'].merge!(file['auth_header'])
|
|
216
|
+
end
|
|
217
|
+
# this only works for uniquely named files
|
|
218
|
+
target_file = File.join(files_path, file['file_name'].tr(' ', '_'))
|
|
219
|
+
target_files << target_file
|
|
220
|
+
# Now because we want the files in place before the importer runs
|
|
221
|
+
# Problematic for a large upload
|
|
222
|
+
Bulkrax::DownloadCloudFileJob.perform_later(file, target_file)
|
|
250
223
|
end
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
# that join would otherwise neutralize a leading `/` (File.join
|
|
254
|
-
# doesn't treat it as absolute once it's not the first argument).
|
|
255
|
-
Bulkrax::CloudSourceValidator.preflight!(file)
|
|
256
|
-
|
|
257
|
-
# this only works for uniquely named files
|
|
258
|
-
relative_filename = File.join('files', file['file_name'].tr(' ', '_'))
|
|
259
|
-
target_file = Bulkrax::SafePath.confine!(path_for_import, relative_filename, require_file: true)
|
|
260
|
-
[file, relative_filename, target_file]
|
|
224
|
+
importer[:parser_fields]['original_file_paths'] = target_files
|
|
225
|
+
return nil
|
|
261
226
|
end
|
|
262
227
|
|
|
263
228
|
# export methods
|
|
@@ -403,12 +368,17 @@ module Bulkrax
|
|
|
403
368
|
raise StandardError, "Record references local files but no files directory could be resolved from the import path" if files_dir.nil?
|
|
404
369
|
|
|
405
370
|
r[file_mapping].split(split_pattern).map do |f|
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
371
|
+
reference = f.strip
|
|
372
|
+
# Try the reference as written first, then the space->underscore
|
|
373
|
+
# variant produced by #remove_spaces_from_filenames. The rename only
|
|
374
|
+
# rewrites the top level of files/ (see unzip_spec), so nested
|
|
375
|
+
# attachments keep their spaces on disk and only match as written.
|
|
376
|
+
file = [reference, reference.tr(' ', '_')].uniq
|
|
377
|
+
.map { |candidate| File.join(files_dir, candidate) }
|
|
378
|
+
.find { |candidate| File.exist?(candidate) }
|
|
379
|
+
raise "File #{File.join(files_dir, reference)} does not exist" if file.nil?
|
|
380
|
+
|
|
381
|
+
file
|
|
412
382
|
end
|
|
413
383
|
end.flatten.compact.uniq
|
|
414
384
|
end
|