bulkrax 8.2.4 → 8.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/javascripts/bulkrax/bulkrax.js +9 -7
- data/app/assets/javascripts/bulkrax/importers.js.erb +134 -99
- data/app/controllers/bulkrax/importers_controller.rb +28 -112
- data/app/controllers/concerns/bulkrax/datatables_behavior.rb +1 -1
- data/app/factories/bulkrax/object_factory.rb +1 -4
- data/app/jobs/bulkrax/download_cloud_file_job.rb +21 -156
- data/app/jobs/bulkrax/export_work_job.rb +1 -1
- data/app/models/bulkrax/status.rb +1 -1
- data/app/models/concerns/bulkrax/export_behavior.rb +4 -8
- data/app/parsers/bulkrax/application_parser.rb +2 -25
- data/app/parsers/bulkrax/bagit_parser.rb +3 -10
- data/app/parsers/bulkrax/csv_parser.rb +17 -51
- data/app/parsers/bulkrax/parser_export_record_set.rb +1 -1
- data/app/views/bulkrax/importers/_bagit_fields.html.erb +10 -3
- data/app/views/bulkrax/importers/_csv_fields.html.erb +13 -9
- data/app/views/bulkrax/importers/_file_uploader.html.erb +37 -0
- data/app/views/bulkrax/importers/_xml_fields.html.erb +10 -4
- data/config/locales/bulkrax.en.yml +0 -3
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +0 -1
- data/lib/tasks/bulkrax_tasks.rake +1 -1
- metadata +9 -32
- data/app/errors/bulkrax/unsafe_cloud_source_error.rb +0 -11
- data/app/errors/bulkrax/unsafe_path_error.rb +0 -8
- data/app/parsers/bulkrax/application_parser.rb.orig +0 -545
- data/app/services/bulkrax/cloud_source_validator.rb +0 -123
- data/app/services/bulkrax/safe_path.rb +0 -142
- data/db/migrate/20241203010707_entry_error_denormalization.rb +0 -7
- data/db/migrate/20241205212513_faster_first_entry.rb +0 -7
|
@@ -1,166 +1,31 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
|
-
|
|
3
2
|
module Bulkrax
|
|
4
3
|
class DownloadCloudFileJob < ApplicationJob
|
|
5
4
|
queue_as Bulkrax.config.ingest_queue_name
|
|
6
5
|
|
|
7
|
-
|
|
8
|
-
#
|
|
9
|
-
# `importer_id:` and `relative_filename:` are required keywords: the
|
|
10
|
-
# destination is always recomputed here, confined to this importer's
|
|
11
|
-
# own workspace, rather than trusted from a serialized job argument.
|
|
12
|
-
# `target_file` is a deprecated positional placeholder, kept only so
|
|
13
|
-
# that a pre-upgrade call shape (`file`, `target_file`) still binds
|
|
14
|
-
# without Ruby raising its own ArgumentError — it is never read, even
|
|
15
|
-
# when supplied alongside otherwise-valid keywords, so it can't be
|
|
16
|
-
# used to redirect the destination. A call made this way is missing
|
|
17
|
-
# the required keywords, so it raises below rather than running.
|
|
18
|
-
#
|
|
19
|
-
# Note: if using the file system provider, the mounted directory in
|
|
20
|
-
# browse_everything MUST be shared by web and worker servers
|
|
21
|
-
def perform(file, target_file = nil, importer_id: nil, relative_filename: nil)
|
|
22
|
-
raise Bulkrax::UnsafePathError, I18n.t('bulkrax.importer.browse_everything.errors.missing_destination_context') if importer_id.blank? || relative_filename.blank?
|
|
23
|
-
|
|
24
|
-
# `file` and `relative_filename` are independent arguments and may
|
|
25
|
-
# not agree (a tampered or replayed job call could supply either
|
|
26
|
-
# one separately) — check the raw file_name directly rather than
|
|
27
|
-
# assuming it was already validated by whatever enqueued this job.
|
|
28
|
-
Bulkrax::SafePath.reject_unsafe_entry!(file['file_name'])
|
|
29
|
-
|
|
30
|
-
importer = Bulkrax::Importer.find(importer_id)
|
|
31
|
-
target_file = Bulkrax::SafePath.confine!(importer.parser.path_for_import, relative_filename, require_file: true)
|
|
32
|
-
|
|
33
|
-
# Re-validate the source here too: this job may run on a different
|
|
34
|
-
# process/host than the one that enqueued it, and could be invoked
|
|
35
|
-
# directly with a tampered or stale `file` argument.
|
|
36
|
-
Bulkrax::CloudSourceValidator.validate!(file)
|
|
37
|
-
|
|
38
|
-
download(file, target_file)
|
|
39
|
-
Rails.logger.info "Bulkrax: cloud file download complete for #{file['file_name']}"
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
private
|
|
43
|
-
|
|
44
|
-
# Downloads through a temp file in the *same* directory as
|
|
45
|
-
# `target_file` (so the final move is same-filesystem and atomic),
|
|
46
|
-
# named with a random suffix rather than anything derived from
|
|
47
|
-
# caller-controlled input, and removed on any failure. This means a
|
|
48
|
-
# failed or non-success download can never truncate or corrupt an
|
|
49
|
-
# already-good destination file from a previous successful import.
|
|
50
|
-
def download(file, target_file)
|
|
51
|
-
temp_file = "#{target_file}.#{SecureRandom.hex(8)}.part"
|
|
52
|
-
uri = Addressable::URI.parse(file['url'].to_s)
|
|
53
|
-
if uri.scheme == 'file'
|
|
54
|
-
download_file_source(uri, temp_file)
|
|
55
|
-
else
|
|
56
|
-
download_http_source(file, URI.parse(file['url'].to_s), temp_file)
|
|
57
|
-
end
|
|
58
|
-
FileUtils.mv(temp_file, target_file)
|
|
59
|
-
ensure
|
|
60
|
-
FileUtils.rm_f(temp_file) if temp_file && File.exist?(temp_file)
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
# The source was already confined to the configured filesystem
|
|
64
|
-
# provider's root by {Bulkrax::CloudSourceValidator}.
|
|
65
|
-
def download_file_source(uri, temp_file)
|
|
66
|
-
source_path = URI::DEFAULT_PARSER.unescape(uri.path.to_s)
|
|
67
|
-
IO.copy_stream(source_path, temp_file)
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
# Resolved, IP-pinned, redirect-free, and streamed straight to disk in
|
|
71
|
-
# bounded-size chunks — rather than handing the URL back to a transport
|
|
72
|
-
# (browse-everything's Retriever) that re-resolves the hostname with no
|
|
73
|
-
# SSRF protection at all. This is a *separate* connection from the one
|
|
74
|
-
# {Bulkrax::CloudSourceValidator}'s HEAD check made; each independently
|
|
75
|
-
# applies the same validation at its own connection time, so there is
|
|
76
|
-
# no single "pinned connection" shared between validation and download.
|
|
77
|
-
#
|
|
78
|
-
# ssrf_filter 1.0.8's public `get`/`head` API has no response-streaming
|
|
79
|
-
# hook: `fetch_once` calls `Net::HTTP#request` with no block, which
|
|
80
|
-
# reads the entire response body into memory before returning; the
|
|
81
|
-
# block `get`/`head` accept is for customizing the *outgoing* request,
|
|
82
|
-
# not consuming the response. So we can't use `SsrfFilter.get` here
|
|
83
|
-
# without buffering the whole remote file in memory first.
|
|
84
|
-
#
|
|
85
|
-
# Reuse ssrf_filter's resolver and IP policy, then pin the validated
|
|
86
|
-
# address with Net::HTTP#ipaddr=. Keeping the original hostname on
|
|
87
|
-
# the HTTP connection preserves native TLS SNI and certificate
|
|
88
|
-
# verification without relying on ssrf_filter's TLS monkeypatches.
|
|
89
|
-
def download_http_source(file, uri, temp_file)
|
|
90
|
-
headers = Bulkrax::CloudSourceValidator.request_headers(file)
|
|
91
|
-
stream_download(uri, headers) { |response| handle_response(file, response, temp_file) }
|
|
92
|
-
rescue SsrfFilter::Error
|
|
93
|
-
# The validator's own HEAD check already ran, but DNS can change
|
|
94
|
-
# between that check and this GET — same rationale as
|
|
95
|
-
# Bulkrax::CloudSourceValidator#validate_http_scheme!, and the same
|
|
96
|
-
# static, class-agnostic message for the same reason.
|
|
97
|
-
raise Bulkrax::UnsafeCloudSourceError, 'blocked unsafe remote source'
|
|
98
|
-
end
|
|
6
|
+
include ActionView::Helpers::NumberHelper
|
|
99
7
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
# as the `Host` header and used for TLS SNI/certificate verification
|
|
120
|
-
# instead of the IP. Yields the live `Net::HTTPResponse` to the block
|
|
121
|
-
# *before* its body is read, so the caller can stream it.
|
|
122
|
-
def stream_download(uri, headers)
|
|
123
|
-
raise SsrfFilter::InvalidUriScheme, "URI scheme '#{uri.scheme}' not in whitelist" unless
|
|
124
|
-
SsrfFilter::DEFAULT_SCHEME_WHITELIST.include?(uri.scheme)
|
|
125
|
-
|
|
126
|
-
hostname = uri.hostname
|
|
127
|
-
ip_addresses = SsrfFilter::DEFAULT_RESOLVER.call(hostname)
|
|
128
|
-
raise SsrfFilter::UnresolvedHostname, "Could not resolve hostname '#{hostname}'" if ip_addresses.empty?
|
|
129
|
-
|
|
130
|
-
public_addresses = ip_addresses.reject { |ip| unsafe_ip_address?(ip) }
|
|
131
|
-
raise SsrfFilter::PrivateIPAddress, "Hostname '#{hostname}' has no public ip addresses" if public_addresses.empty?
|
|
132
|
-
|
|
133
|
-
request = Net::HTTP::Get.new(uri)
|
|
134
|
-
request['host'] = host_header(hostname, uri)
|
|
135
|
-
headers.each { |key, value| request[key] = value }
|
|
136
|
-
|
|
137
|
-
http = Net::HTTP.new(hostname, uri.port, nil)
|
|
138
|
-
http.ipaddr = public_addresses.sample.to_s
|
|
139
|
-
http.use_ssl = (uri.scheme == 'https')
|
|
140
|
-
http.start { |connection| connection.request(request) { |response| yield response } }
|
|
141
|
-
end
|
|
142
|
-
|
|
143
|
-
# Mirrors `SsrfFilter.unsafe_ip_address?`, but against its public
|
|
144
|
-
# `IPV4_BLACKLIST`/`IPV6_BLACKLIST` constants directly rather than
|
|
145
|
-
# calling that private method — we only ever check single resolved
|
|
146
|
-
# addresses here (never a CIDR range), so the masked-range guard it
|
|
147
|
-
# also carries doesn't apply.
|
|
148
|
-
def unsafe_ip_address?(ip_address)
|
|
149
|
-
return SsrfFilter::IPV4_BLACKLIST.any? { |range| range.include?(ip_address) } if ip_address.ipv4?
|
|
150
|
-
return SsrfFilter::IPV6_BLACKLIST.any? { |range| range.include?(ip_address) } if ip_address.ipv6?
|
|
151
|
-
|
|
152
|
-
true
|
|
153
|
-
end
|
|
154
|
-
|
|
155
|
-
# Matches `SsrfFilter.host_header`: the `Host` header should carry the
|
|
156
|
-
# real hostname (not the pinned IP we actually connect to), with the
|
|
157
|
-
# port suffixed only when it isn't the scheme's default.
|
|
158
|
-
def host_header(hostname, uri)
|
|
159
|
-
if (uri.port == 80 && uri.scheme == 'http') || (uri.port == 443 && uri.scheme == 'https')
|
|
160
|
-
hostname
|
|
161
|
-
else
|
|
162
|
-
"#{hostname}:#{uri.port}"
|
|
8
|
+
# Retrieve cloud file and write to the imports directory
|
|
9
|
+
# Note: if using the file system, the mounted directory in
|
|
10
|
+
# browse_everything MUST be shared by web and worker servers
|
|
11
|
+
def perform(file, target_file)
|
|
12
|
+
retriever = BrowseEverything::Retriever.new
|
|
13
|
+
last_logged_time = Time.zone.now
|
|
14
|
+
log_interval = 3.seconds
|
|
15
|
+
|
|
16
|
+
retriever.download(file, target_file) do |filename, retrieved, total|
|
|
17
|
+
percentage = (retrieved.to_f / total.to_f) * 100
|
|
18
|
+
current_time = Time.zone.now
|
|
19
|
+
|
|
20
|
+
if (current_time - last_logged_time) >= log_interval
|
|
21
|
+
# Use number_to_human_size for formatting
|
|
22
|
+
readable_retrieved = number_to_human_size(retrieved)
|
|
23
|
+
readable_total = number_to_human_size(total)
|
|
24
|
+
Rails.logger.info "Downloaded #{readable_retrieved} of #{readable_total}, #{filename}: #{percentage.round}% complete"
|
|
25
|
+
last_logged_time = current_time
|
|
26
|
+
end
|
|
163
27
|
end
|
|
28
|
+
Rails.logger.info "Download complete: #{file['url']} to #{target_file}"
|
|
164
29
|
end
|
|
165
30
|
end
|
|
166
31
|
end
|
|
@@ -26,7 +26,7 @@ module Bulkrax
|
|
|
26
26
|
end
|
|
27
27
|
# rubocop:enable Rails/SkipsModelValidations
|
|
28
28
|
end
|
|
29
|
-
return entry if exporter_run.
|
|
29
|
+
return entry if exporter_run.enqueued_records.positive?
|
|
30
30
|
|
|
31
31
|
if exporter_run.failed_records.positive?
|
|
32
32
|
exporter_run.exporter.set_status_info('Complete (with failures)')
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Bulkrax
|
|
4
4
|
class Status < ApplicationRecord
|
|
5
|
-
belongs_to :statusable, polymorphic: true, denormalize: { fields: %i[status_message
|
|
5
|
+
belongs_to :statusable, polymorphic: true, denormalize: { fields: %i[status_message], if: :latest? }
|
|
6
6
|
belongs_to :runnable, polymorphic: true
|
|
7
7
|
serialize :error_backtrace, Array
|
|
8
8
|
|
|
@@ -26,15 +26,11 @@ module Bulkrax
|
|
|
26
26
|
|
|
27
27
|
# Prepend the file_set id to ensure a unique filename and also one that is not longer than 255 characters
|
|
28
28
|
def filename(file_set)
|
|
29
|
+
# NOTE: Will this work with Valkyrie?
|
|
29
30
|
return if file_set.original_file.blank?
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
else # original non valkyrie version
|
|
34
|
-
fn = file_set.original_file.file_name.first
|
|
35
|
-
mime = ::Marcel::MimeType.for(declared_type: file_set.original_file.mime_type)
|
|
36
|
-
end
|
|
37
|
-
ext_mime = ::Marcel::MimeType.for(name: fn)
|
|
31
|
+
fn = file_set.original_file.file_name.first
|
|
32
|
+
mime = ::Marcel::MimeType.for(file_set.original_file.mime_type)
|
|
33
|
+
ext_mime = ::Marcel::MimeType.for(file_set.original_file.file_name)
|
|
38
34
|
if fn.include?(file_set.id) || importerexporter.metadata_only?
|
|
39
35
|
filename = "#{fn}.#{mime.to_sym}"
|
|
40
36
|
filename = fn if mime.to_s == ext_mime.to_s
|
|
@@ -430,38 +430,15 @@ module Bulkrax
|
|
|
430
430
|
def unzip(file_to_unzip)
|
|
431
431
|
return untar(file_to_unzip) if file_to_unzip.end_with?('.tar.gz')
|
|
432
432
|
|
|
433
|
-
dest_dir = importer_unzip_path
|
|
434
433
|
Zip::File.open(file_to_unzip) do |zip_file|
|
|
435
434
|
zip_file.each do |entry|
|
|
436
|
-
|
|
437
|
-
# for cloud-file destinations, rather than a bare File.join of
|
|
438
|
-
# attacker-controlled entry.name.
|
|
439
|
-
entry_path = Bulkrax::SafePath.confine!(dest_dir, entry.name)
|
|
435
|
+
entry_path = File.join(importer_unzip_path, entry.name)
|
|
440
436
|
FileUtils.mkdir_p(File.dirname(entry_path))
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
extract_zip_entry(zip_file, entry, dest_dir, entry.name, entry_path)
|
|
437
|
+
zip_file.extract(entry, entry_path) unless File.exist?(entry_path)
|
|
444
438
|
end
|
|
445
439
|
end
|
|
446
440
|
end
|
|
447
441
|
|
|
448
|
-
# rubyzip 2.x: extract(entry, absolute_dest_path)
|
|
449
|
-
# rubyzip 3.x: extract(entry, relative_name, destination_directory: dir)
|
|
450
|
-
#
|
|
451
|
-
# entry_path (from Bulkrax::SafePath.confine!) is always an absolute
|
|
452
|
-
# path; rubyzip 3.x's 2-positional-arg #extract treats its second
|
|
453
|
-
# argument as a name relative to the current directory rather than a
|
|
454
|
-
# destination path, which silently re-joins an already-absolute path
|
|
455
|
-
# onto the cwd. The keyword form is the only one both versions can be
|
|
456
|
-
# trusted to interpret as "this absolute/relative destination".
|
|
457
|
-
def extract_zip_entry(zip_file, entry, dest_dir, relative_name, dest_path)
|
|
458
|
-
if zip_file.method(:extract).arity == 2
|
|
459
|
-
zip_file.extract(entry, dest_path)
|
|
460
|
-
else
|
|
461
|
-
zip_file.extract(entry, relative_name, destination_directory: dest_dir)
|
|
462
|
-
end
|
|
463
|
-
end
|
|
464
|
-
|
|
465
442
|
def untar(file_to_untar)
|
|
466
443
|
Dir.mkdir(importer_unzip_path) unless File.directory?(importer_unzip_path)
|
|
467
444
|
command = "tar -xzf #{Shellwords.escape(file_to_untar)} -C #{Shellwords.escape(importer_unzip_path)}"
|
|
@@ -166,19 +166,12 @@ unless ENV.fetch('BULKRAX_NO_BAGIT', 'false').to_s == 'true'
|
|
|
166
166
|
# @todo - investigate getting directory structure
|
|
167
167
|
# @todo - investigate using perform_later, and having the importer check for
|
|
168
168
|
# DownloadCloudFileJob before it starts
|
|
169
|
-
def retrieve_cloud_files(files,
|
|
169
|
+
def retrieve_cloud_files(files, _importer)
|
|
170
170
|
# There should only be one zip file for Bagit, take the first
|
|
171
171
|
return if files['0'].blank?
|
|
172
|
-
|
|
173
|
-
file = files['0']
|
|
174
|
-
# Checked on the raw file_name before any normalization — see
|
|
175
|
-
# Bulkrax::CsvParser#plan_cloud_file for why this must come first.
|
|
176
|
-
Bulkrax::CloudSourceValidator.preflight!(file)
|
|
177
|
-
|
|
178
|
-
relative_filename = file['file_name'].tr(' ', '_')
|
|
179
|
-
target_file = Bulkrax::SafePath.confine!(path_for_import, relative_filename, require_file: true)
|
|
172
|
+
target_file = File.join(path_for_import, files['0']['file_name'].tr(' ', '_'))
|
|
180
173
|
# Now because we want the files in place before the importer runs
|
|
181
|
-
Bulkrax::DownloadCloudFileJob.perform_now(
|
|
174
|
+
Bulkrax::DownloadCloudFileJob.perform_now(files['0'], target_file)
|
|
182
175
|
return target_file
|
|
183
176
|
end
|
|
184
177
|
|
|
@@ -194,55 +194,24 @@ module Bulkrax
|
|
|
194
194
|
# @todo - investigate using perform_later, and having the importer check for
|
|
195
195
|
# DownloadCloudFileJob before it starts
|
|
196
196
|
def retrieve_cloud_files(files, importer)
|
|
197
|
-
plan = plan_cloud_files(files)
|
|
198
|
-
dispatch_cloud_files!(plan, importer)
|
|
199
|
-
importer[:parser_fields]['original_file_paths'] = plan.map { |_file, _relative_filename, target_file| target_file }
|
|
200
|
-
return nil
|
|
201
|
-
end
|
|
202
|
-
|
|
203
|
-
# Validates and resolves the whole batch, without enqueueing anything
|
|
204
|
-
# yet. Otherwise an earlier, valid file's job would already be running
|
|
205
|
-
# by the time a later file in the same batch is found to be unsafe.
|
|
206
|
-
# Split out from {#retrieve_cloud_files} so a caller (the importers
|
|
207
|
-
# controller) can persist the resolved paths before dispatching.
|
|
208
|
-
#
|
|
209
|
-
# @return [Array<Array(Hash, String, String)>] one entry per file: the
|
|
210
|
-
# file spec, its relative destination filename, and the resolved
|
|
211
|
-
# absolute target path
|
|
212
|
-
def plan_cloud_files(files)
|
|
213
197
|
files_path = File.join(path_for_import, 'files')
|
|
214
198
|
FileUtils.mkdir_p(files_path) unless File.exist?(files_path)
|
|
215
|
-
|
|
216
|
-
files.
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
# @return [Array(Hash, String, String)] the file spec, its relative
|
|
229
|
-
# destination filename, and the resolved absolute target path
|
|
230
|
-
def plan_cloud_file(file)
|
|
231
|
-
# fixes bug where auth headers do not get attached properly
|
|
232
|
-
if file['auth_header'].present?
|
|
233
|
-
file['headers'] ||= {}
|
|
234
|
-
file['headers'].merge!(file['auth_header'])
|
|
199
|
+
target_files = []
|
|
200
|
+
files.each_pair do |_key, file|
|
|
201
|
+
# fixes bug where auth headers do not get attached properly
|
|
202
|
+
if file['auth_header'].present?
|
|
203
|
+
file['headers'] ||= {}
|
|
204
|
+
file['headers'].merge!(file['auth_header'])
|
|
205
|
+
end
|
|
206
|
+
# this only works for uniquely named files
|
|
207
|
+
target_file = File.join(files_path, file['file_name'].tr(' ', '_'))
|
|
208
|
+
target_files << target_file
|
|
209
|
+
# Now because we want the files in place before the importer runs
|
|
210
|
+
# Problematic for a large upload
|
|
211
|
+
Bulkrax::DownloadCloudFileJob.perform_later(file, target_file)
|
|
235
212
|
end
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
# that join would otherwise neutralize a leading `/` (File.join
|
|
239
|
-
# doesn't treat it as absolute once it's not the first argument).
|
|
240
|
-
Bulkrax::CloudSourceValidator.preflight!(file)
|
|
241
|
-
|
|
242
|
-
# this only works for uniquely named files
|
|
243
|
-
relative_filename = File.join('files', file['file_name'].tr(' ', '_'))
|
|
244
|
-
target_file = Bulkrax::SafePath.confine!(path_for_import, relative_filename, require_file: true)
|
|
245
|
-
[file, relative_filename, target_file]
|
|
213
|
+
importer[:parser_fields]['original_file_paths'] = target_files
|
|
214
|
+
return nil
|
|
246
215
|
end
|
|
247
216
|
|
|
248
217
|
# export methods
|
|
@@ -275,10 +244,7 @@ module Bulkrax
|
|
|
275
244
|
record = Bulkrax.object_factory.find(identifier)
|
|
276
245
|
return unless record
|
|
277
246
|
|
|
278
|
-
file_sets = Array.wrap(record)
|
|
279
|
-
if file_sets.nil? # for valkyrie
|
|
280
|
-
file_sets = record.respond_to?(:file_sets) ? record.file_sets : record.members&.select(&:file_set?)
|
|
281
|
-
end
|
|
247
|
+
file_sets = record.file_set? ? Array.wrap(record) : record.file_sets
|
|
282
248
|
file_sets << record.thumbnail if exporter.include_thumbnails && record.thumbnail.present? && record.work?
|
|
283
249
|
file_sets.each do |fs|
|
|
284
250
|
path = File.join(exporter_export_path, folder_count, 'files')
|
|
@@ -286,7 +252,7 @@ module Bulkrax
|
|
|
286
252
|
file = filename(fs)
|
|
287
253
|
next if file.blank? || fs.original_file.blank?
|
|
288
254
|
|
|
289
|
-
io =
|
|
255
|
+
io = open(fs.original_file.uri)
|
|
290
256
|
File.open(File.join(path, file), 'wb') do |f|
|
|
291
257
|
f.write(io.read)
|
|
292
258
|
f.close
|
|
@@ -173,7 +173,7 @@ module Bulkrax
|
|
|
173
173
|
# @see https://github.com/samvera/hyrax/blob/64c0bbf0dc0d3e1b49f040b50ea70d177cc9d8f6/app/indexers/hyrax/work_indexer.rb#L15-L18
|
|
174
174
|
def file_sets
|
|
175
175
|
@file_sets ||= ParserExportRecordSet.in_batches(candidate_file_set_ids) do |batch_of_ids|
|
|
176
|
-
fsq = "has_model_ssim
|
|
176
|
+
fsq = "has_model_ssim:#{Bulkrax.file_model_internal_resource} AND id:(\"" + batch_of_ids.join('" OR "') + "\")"
|
|
177
177
|
fsq += extra_filters if extra_filters.present?
|
|
178
178
|
Bulkrax.object_factory.query(
|
|
179
179
|
fsq,
|
|
@@ -41,9 +41,16 @@
|
|
|
41
41
|
<p>File upload and Cloud File upload must be a Zip file containing a single BagIt Bag, or a folder containing multiple BagIt Bags.</p>
|
|
42
42
|
<p>The Server Path can point to a BagIt Bag, a folder containing BagIt Bags, or a zip file containing either.</p>
|
|
43
43
|
|
|
44
|
-
|
|
44
|
+
<%= fi.input :file_style,
|
|
45
|
+
collection: ['Upload a File', 'Specify a Path on the Server'] +
|
|
46
|
+
(defined?(::Hyrax) && Hyrax.config.browse_everything? ? ['Add Cloud File'] : []),
|
|
47
|
+
as: :radio_buttons, label: false %>
|
|
45
48
|
<div id='file_upload'>
|
|
46
|
-
|
|
49
|
+
<% if defined?(::Hyrax) %>
|
|
50
|
+
<%= render 'bulkrax/importers/file_uploader', accepted_file_types: 'application/zip' %>
|
|
51
|
+
<% else %>
|
|
52
|
+
<%= fi.input 'file', as: :file, input_html: {accept: 'application/zip'} %><br />
|
|
53
|
+
<% end %>
|
|
47
54
|
</div>
|
|
48
55
|
<div id='file_path'>
|
|
49
56
|
<%= fi.input :import_file_path, as: :string, input_html: { value: importer.parser_fields['import_file_path'] } %>
|
|
@@ -53,4 +60,4 @@
|
|
|
53
60
|
<%= render 'browse_everything', form: form %>
|
|
54
61
|
<% end %>
|
|
55
62
|
</div>
|
|
56
|
-
</div>
|
|
63
|
+
</div>
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
<div class='csv_fields'>
|
|
2
|
-
|
|
3
2
|
<%= fi.input :visibility,
|
|
4
3
|
label: 'Default Visibility',
|
|
5
4
|
collection: [
|
|
@@ -11,7 +10,6 @@
|
|
|
11
10
|
input_html: { class: 'form-control' },
|
|
12
11
|
hint: 'If your CSV includes the visibility field, it will override the default setting.'
|
|
13
12
|
%>
|
|
14
|
-
|
|
15
13
|
<% if defined?(::Hyrax) %>
|
|
16
14
|
<% rights_statements = Hyrax.config.rights_statement_service_class.new %>
|
|
17
15
|
<%= fi.input :rights_statement,
|
|
@@ -24,26 +22,32 @@
|
|
|
24
22
|
%>
|
|
25
23
|
<%= fi.input :override_rights_statement, as: :boolean, hint: 'If checked, always use the selected rights statment. If unchecked, use rights or rights_statement from the record and only use the provided value if dc:rights is blank.', input_html: { checked: (importer.parser_fields['override_rights_statement'] == "1") } %>
|
|
26
24
|
<% end %>
|
|
27
|
-
<h4>Add CSV File to Import:</h4>
|
|
25
|
+
<h4>Add CSV or ZIP File to Import:</h4>
|
|
28
26
|
<%# accept a single file upload; data files and bags will need to be added another way %>
|
|
29
|
-
|
|
30
27
|
<% file_style_list = ['Upload a File', 'Specify a Path on the Server'] %>
|
|
31
28
|
<% file_style_list << 'Existing Entries' unless importer.new_record? %>
|
|
32
29
|
<%= fi.input :file_style, collection: file_style_list, as: :radio_buttons, label: false %>
|
|
30
|
+
|
|
33
31
|
<div id='file_upload'>
|
|
34
|
-
|
|
32
|
+
<% if defined?(::Hyrax) %>
|
|
33
|
+
<%= render 'bulkrax/importers/file_uploader', accepted_file_types: 'text/csv,application/zip,application/gzip' %>
|
|
34
|
+
<% else %>
|
|
35
|
+
<%= fi.input 'file', as: :file, input_html: { accept: 'text/csv,application/zip,application/gzip' } %><br />
|
|
36
|
+
<% end %>
|
|
35
37
|
</div>
|
|
38
|
+
|
|
36
39
|
<div id='file_path'>
|
|
37
40
|
<%= fi.input :import_file_path, as: :string, input_html: { value: importer.parser_fields['import_file_path'] } %>
|
|
38
41
|
</div>
|
|
42
|
+
|
|
39
43
|
<div id='existing_options'>
|
|
40
44
|
<%= fi.collection_check_boxes :entry_statuses, [['Failed'], ['Pending'], ['Skipped'], ['Deleted'], ['Complete']], :first, :first %>
|
|
41
45
|
</div>
|
|
42
|
-
|
|
46
|
+
|
|
43
47
|
<% if defined?(::Hyrax) && Hyrax.config.browse_everything? %>
|
|
44
|
-
|
|
45
|
-
|
|
48
|
+
<h4>Add Files to Import:</h4>
|
|
49
|
+
<p>Choose files to upload. The filenames must be unique, and the filenames must be referenced in a column called 'file' in the accompanying CSV file.</p>
|
|
46
50
|
<%= render 'browse_everything', form: form %>
|
|
47
51
|
<% end %>
|
|
48
52
|
<br />
|
|
49
|
-
</div>
|
|
53
|
+
</div>
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
<div class="fileupload-bulkrax">
|
|
2
|
+
<noscript><input type="hidden" name="redirect" value="<%= main_app.root_path %>" /></noscript>
|
|
3
|
+
<table role="presentation" class="table table-striped"><tbody class="files"></tbody></table>
|
|
4
|
+
<div class="fileupload-buttonbar">
|
|
5
|
+
<div class="row">
|
|
6
|
+
<div class="col-12">
|
|
7
|
+
<div class="fileinput-button" id="add-files">
|
|
8
|
+
<input id="addfiles" type="file" style="display:none;" name="files[]"
|
|
9
|
+
accept="<%= accepted_file_types || 'text/csv,application/zip,application/gzip' %>" multiple />
|
|
10
|
+
<button type="button" class="btn btn-success" onclick="document.getElementById('addfiles').click();">
|
|
11
|
+
<span class="fa fa-plus"></span>
|
|
12
|
+
<span>Add Files</span>
|
|
13
|
+
</button>
|
|
14
|
+
</div>
|
|
15
|
+
<button type="reset" id="file-upload-cancel-btn" class="btn btn-warning cancel">
|
|
16
|
+
<span class="fa fa-ban"></span>
|
|
17
|
+
<span>Cancel Upload</span>
|
|
18
|
+
</button>
|
|
19
|
+
<span class="fileupload-process"></span>
|
|
20
|
+
</div>
|
|
21
|
+
</div>
|
|
22
|
+
<div class="row">
|
|
23
|
+
<div class="col-12">
|
|
24
|
+
<div class="fileupload-progress fade">
|
|
25
|
+
<div class="progress" role="progressbar" aria-valuemin="0" aria-valuemax="100">
|
|
26
|
+
<div class="progress-bar progress-bar-striped progress-bar-animated bg-success" style="width:0%;"></div>
|
|
27
|
+
</div>
|
|
28
|
+
<div class="progress-extended"> </div>
|
|
29
|
+
</div>
|
|
30
|
+
</div>
|
|
31
|
+
</div>
|
|
32
|
+
</div>
|
|
33
|
+
<div class="dropzone">
|
|
34
|
+
Drop files here to upload
|
|
35
|
+
</div>
|
|
36
|
+
<%= render 'hyrax/uploads/js_templates' %>
|
|
37
|
+
</div>
|
|
@@ -47,16 +47,22 @@
|
|
|
47
47
|
<p>File upload and Cloud File upload MUST be a either a single XML file (for metadata only import) OR a Zip file containing the XML files and data files, each in a separate folder.</p>
|
|
48
48
|
<p>The Server Path can point to a folder containing XML files and data files to import, or direct to the XML file itself.</p>
|
|
49
49
|
|
|
50
|
-
|
|
50
|
+
<%= fi.input :file_style,
|
|
51
|
+
collection: ['Upload a File', 'Specify a Path on the Server'] +
|
|
52
|
+
(defined?(::Hyrax) && Hyrax.config.browse_everything? ? ['Add Cloud File'] : []),
|
|
53
|
+
as: :radio_buttons, label: false %>
|
|
51
54
|
<div id='file_upload'>
|
|
52
|
-
|
|
55
|
+
<% if defined?(::Hyrax) %>
|
|
56
|
+
<%= render 'bulkrax/importers/file_uploader', accepted_file_types: 'application/zip,application/xml' %>
|
|
57
|
+
<% else %>
|
|
58
|
+
<%= fi.input 'file', as: :file, input_html: {accept: ['application/zip', 'application/xml']} %><br />
|
|
59
|
+
<% end%>
|
|
53
60
|
</div>
|
|
54
61
|
<div id='file_path'>
|
|
55
62
|
<%= fi.input :import_file_path, as: :string, input_html: { value: importer.parser_fields['import_file_path'] } %>
|
|
56
63
|
</div>
|
|
57
64
|
<div id='cloud'>
|
|
58
65
|
<% if defined?(::Hyrax) && Hyrax.config.browse_everything? %>
|
|
59
|
-
<%= render 'browse_everything', form: form %>
|
|
60
66
|
<% end %>
|
|
61
67
|
</div>
|
|
62
|
-
</div>
|
|
68
|
+
</div>
|
|
@@ -70,9 +70,6 @@ en:
|
|
|
70
70
|
type: Type
|
|
71
71
|
user: User
|
|
72
72
|
work_entries: Work Entries
|
|
73
|
-
browse_everything:
|
|
74
|
-
errors:
|
|
75
|
-
missing_destination_context: This cloud file download is missing required destination information and can no longer be safely processed. Please re-select the file and try the import again.
|
|
76
73
|
table_header:
|
|
77
74
|
labels:
|
|
78
75
|
identifier: Identifier
|
data/lib/bulkrax/version.rb
CHANGED
data/lib/bulkrax.rb
CHANGED
|
@@ -9,7 +9,7 @@ namespace :bulkrax do
|
|
|
9
9
|
progress_mark: ' ',
|
|
10
10
|
remainder_mark: "\u{FF65}")
|
|
11
11
|
Bulkrax::Status.latest_by_statusable.includes(:statusable).find_each do |status|
|
|
12
|
-
status.statusable.update(status_message: status.status_message
|
|
12
|
+
status.statusable.update(status_message: status.status_message)
|
|
13
13
|
@progress.increment
|
|
14
14
|
end
|
|
15
15
|
end
|