bulkrax 8.2.4 → 8.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. checksums.yaml +4 -4
  2. data/app/assets/javascripts/bulkrax/bulkrax.js +9 -7
  3. data/app/assets/javascripts/bulkrax/importers.js.erb +134 -99
  4. data/app/controllers/bulkrax/importers_controller.rb +28 -112
  5. data/app/controllers/concerns/bulkrax/datatables_behavior.rb +1 -1
  6. data/app/factories/bulkrax/object_factory.rb +1 -4
  7. data/app/jobs/bulkrax/download_cloud_file_job.rb +21 -156
  8. data/app/jobs/bulkrax/export_work_job.rb +1 -1
  9. data/app/models/bulkrax/status.rb +1 -1
  10. data/app/models/concerns/bulkrax/export_behavior.rb +4 -8
  11. data/app/parsers/bulkrax/application_parser.rb +2 -25
  12. data/app/parsers/bulkrax/bagit_parser.rb +3 -10
  13. data/app/parsers/bulkrax/csv_parser.rb +17 -51
  14. data/app/parsers/bulkrax/parser_export_record_set.rb +1 -1
  15. data/app/views/bulkrax/importers/_bagit_fields.html.erb +10 -3
  16. data/app/views/bulkrax/importers/_csv_fields.html.erb +13 -9
  17. data/app/views/bulkrax/importers/_file_uploader.html.erb +37 -0
  18. data/app/views/bulkrax/importers/_xml_fields.html.erb +10 -4
  19. data/config/locales/bulkrax.en.yml +0 -3
  20. data/lib/bulkrax/version.rb +1 -1
  21. data/lib/bulkrax.rb +0 -1
  22. data/lib/tasks/bulkrax_tasks.rake +1 -1
  23. metadata +9 -32
  24. data/app/errors/bulkrax/unsafe_cloud_source_error.rb +0 -11
  25. data/app/errors/bulkrax/unsafe_path_error.rb +0 -8
  26. data/app/parsers/bulkrax/application_parser.rb.orig +0 -545
  27. data/app/services/bulkrax/cloud_source_validator.rb +0 -123
  28. data/app/services/bulkrax/safe_path.rb +0 -142
  29. data/db/migrate/20241203010707_entry_error_denormalization.rb +0 -7
  30. data/db/migrate/20241205212513_faster_first_entry.rb +0 -7
@@ -1,166 +1,31 @@
1
1
  # frozen_string_literal: true
2
-
3
2
  module Bulkrax
4
3
  class DownloadCloudFileJob < ApplicationJob
5
4
  queue_as Bulkrax.config.ingest_queue_name
6
5
 
7
- # Retrieve a cloud file and write it to the importer's workspace.
8
- #
9
- # `importer_id:` and `relative_filename:` are required keywords: the
10
- # destination is always recomputed here, confined to this importer's
11
- # own workspace, rather than trusted from a serialized job argument.
12
- # `target_file` is a deprecated positional placeholder, kept only so
13
- # that a pre-upgrade call shape (`file`, `target_file`) still binds
14
- # without Ruby raising its own ArgumentError — it is never read, even
15
- # when supplied alongside otherwise-valid keywords, so it can't be
16
- # used to redirect the destination. A call made this way is missing
17
- # the required keywords, so it raises below rather than running.
18
- #
19
- # Note: if using the file system provider, the mounted directory in
20
- # browse_everything MUST be shared by web and worker servers
21
- def perform(file, target_file = nil, importer_id: nil, relative_filename: nil)
22
- raise Bulkrax::UnsafePathError, I18n.t('bulkrax.importer.browse_everything.errors.missing_destination_context') if importer_id.blank? || relative_filename.blank?
23
-
24
- # `file` and `relative_filename` are independent arguments and may
25
- # not agree (a tampered or replayed job call could supply either
26
- # one separately) — check the raw file_name directly rather than
27
- # assuming it was already validated by whatever enqueued this job.
28
- Bulkrax::SafePath.reject_unsafe_entry!(file['file_name'])
29
-
30
- importer = Bulkrax::Importer.find(importer_id)
31
- target_file = Bulkrax::SafePath.confine!(importer.parser.path_for_import, relative_filename, require_file: true)
32
-
33
- # Re-validate the source here too: this job may run on a different
34
- # process/host than the one that enqueued it, and could be invoked
35
- # directly with a tampered or stale `file` argument.
36
- Bulkrax::CloudSourceValidator.validate!(file)
37
-
38
- download(file, target_file)
39
- Rails.logger.info "Bulkrax: cloud file download complete for #{file['file_name']}"
40
- end
41
-
42
- private
43
-
44
- # Downloads through a temp file in the *same* directory as
45
- # `target_file` (so the final move is same-filesystem and atomic),
46
- # named with a random suffix rather than anything derived from
47
- # caller-controlled input, and removed on any failure. This means a
48
- # failed or non-success download can never truncate or corrupt an
49
- # already-good destination file from a previous successful import.
50
- def download(file, target_file)
51
- temp_file = "#{target_file}.#{SecureRandom.hex(8)}.part"
52
- uri = Addressable::URI.parse(file['url'].to_s)
53
- if uri.scheme == 'file'
54
- download_file_source(uri, temp_file)
55
- else
56
- download_http_source(file, URI.parse(file['url'].to_s), temp_file)
57
- end
58
- FileUtils.mv(temp_file, target_file)
59
- ensure
60
- FileUtils.rm_f(temp_file) if temp_file && File.exist?(temp_file)
61
- end
62
-
63
- # The source was already confined to the configured filesystem
64
- # provider's root by {Bulkrax::CloudSourceValidator}.
65
- def download_file_source(uri, temp_file)
66
- source_path = URI::DEFAULT_PARSER.unescape(uri.path.to_s)
67
- IO.copy_stream(source_path, temp_file)
68
- end
69
-
70
- # Resolved, IP-pinned, redirect-free, and streamed straight to disk in
71
- # bounded-size chunks — rather than handing the URL back to a transport
72
- # (browse-everything's Retriever) that re-resolves the hostname with no
73
- # SSRF protection at all. This is a *separate* connection from the one
74
- # {Bulkrax::CloudSourceValidator}'s HEAD check made; each independently
75
- # applies the same validation at its own connection time, so there is
76
- # no single "pinned connection" shared between validation and download.
77
- #
78
- # ssrf_filter 1.0.8's public `get`/`head` API has no response-streaming
79
- # hook: `fetch_once` calls `Net::HTTP#request` with no block, which
80
- # reads the entire response body into memory before returning; the
81
- # block `get`/`head` accept is for customizing the *outgoing* request,
82
- # not consuming the response. So we can't use `SsrfFilter.get` here
83
- # without buffering the whole remote file in memory first.
84
- #
85
- # Reuse ssrf_filter's resolver and IP policy, then pin the validated
86
- # address with Net::HTTP#ipaddr=. Keeping the original hostname on
87
- # the HTTP connection preserves native TLS SNI and certificate
88
- # verification without relying on ssrf_filter's TLS monkeypatches.
89
- def download_http_source(file, uri, temp_file)
90
- headers = Bulkrax::CloudSourceValidator.request_headers(file)
91
- stream_download(uri, headers) { |response| handle_response(file, response, temp_file) }
92
- rescue SsrfFilter::Error
93
- # The validator's own HEAD check already ran, but DNS can change
94
- # between that check and this GET — same rationale as
95
- # Bulkrax::CloudSourceValidator#validate_http_scheme!, and the same
96
- # static, class-agnostic message for the same reason.
97
- raise Bulkrax::UnsafeCloudSourceError, 'blocked unsafe remote source'
98
- end
6
+ include ActionView::Helpers::NumberHelper
99
7
 
100
- def handle_response(file, response, temp_file)
101
- unless response.is_a?(Net::HTTPSuccess)
102
- # Static on purpose — this becomes the importer's status message,
103
- # not just an ops log; see Bulkrax::CloudSourceValidator for why
104
- # status codes/response details aren't echoed into it. The status
105
- # code itself is logged (not attacker-controlled, not a credential).
106
- Rails.logger.error("Bulkrax: cloud file download failed for #{file['file_name']} with status #{response.code}")
107
- raise Bulkrax::UnsafeCloudSourceError, 'download failed: non-success response'
108
- end
109
-
110
- File.open(temp_file, 'wb') { |io| response.read_body { |chunk| io.write(chunk) } }
111
- end
112
-
113
- # Resolves `uri`'s host, rejects it unless it has at least one public
114
- # IP (reusing ssrf_filter's own resolver/blacklist — see above), and
115
- # connects directly to that IP: with the environment proxy explicitly
116
- # disabled (`Net::HTTP.new`'s `p_addr` defaults to `:ENV`, which would
117
- # otherwise route through `http_proxy`/`https_proxy` and hand the
118
- # pinned connection to a third party), and with the real hostname sent
119
- # as the `Host` header and used for TLS SNI/certificate verification
120
- # instead of the IP. Yields the live `Net::HTTPResponse` to the block
121
- # *before* its body is read, so the caller can stream it.
122
- def stream_download(uri, headers)
123
- raise SsrfFilter::InvalidUriScheme, "URI scheme '#{uri.scheme}' not in whitelist" unless
124
- SsrfFilter::DEFAULT_SCHEME_WHITELIST.include?(uri.scheme)
125
-
126
- hostname = uri.hostname
127
- ip_addresses = SsrfFilter::DEFAULT_RESOLVER.call(hostname)
128
- raise SsrfFilter::UnresolvedHostname, "Could not resolve hostname '#{hostname}'" if ip_addresses.empty?
129
-
130
- public_addresses = ip_addresses.reject { |ip| unsafe_ip_address?(ip) }
131
- raise SsrfFilter::PrivateIPAddress, "Hostname '#{hostname}' has no public ip addresses" if public_addresses.empty?
132
-
133
- request = Net::HTTP::Get.new(uri)
134
- request['host'] = host_header(hostname, uri)
135
- headers.each { |key, value| request[key] = value }
136
-
137
- http = Net::HTTP.new(hostname, uri.port, nil)
138
- http.ipaddr = public_addresses.sample.to_s
139
- http.use_ssl = (uri.scheme == 'https')
140
- http.start { |connection| connection.request(request) { |response| yield response } }
141
- end
142
-
143
- # Mirrors `SsrfFilter.unsafe_ip_address?`, but against its public
144
- # `IPV4_BLACKLIST`/`IPV6_BLACKLIST` constants directly rather than
145
- # calling that private method — we only ever check single resolved
146
- # addresses here (never a CIDR range), so the masked-range guard it
147
- # also carries doesn't apply.
148
- def unsafe_ip_address?(ip_address)
149
- return SsrfFilter::IPV4_BLACKLIST.any? { |range| range.include?(ip_address) } if ip_address.ipv4?
150
- return SsrfFilter::IPV6_BLACKLIST.any? { |range| range.include?(ip_address) } if ip_address.ipv6?
151
-
152
- true
153
- end
154
-
155
- # Matches `SsrfFilter.host_header`: the `Host` header should carry the
156
- # real hostname (not the pinned IP we actually connect to), with the
157
- # port suffixed only when it isn't the scheme's default.
158
- def host_header(hostname, uri)
159
- if (uri.port == 80 && uri.scheme == 'http') || (uri.port == 443 && uri.scheme == 'https')
160
- hostname
161
- else
162
- "#{hostname}:#{uri.port}"
8
+ # Retrieve cloud file and write to the imports directory
9
+ # Note: if using the file system, the mounted directory in
10
+ # browse_everything MUST be shared by web and worker servers
11
+ def perform(file, target_file)
12
+ retriever = BrowseEverything::Retriever.new
13
+ last_logged_time = Time.zone.now
14
+ log_interval = 3.seconds
15
+
16
+ retriever.download(file, target_file) do |filename, retrieved, total|
17
+ percentage = (retrieved.to_f / total.to_f) * 100
18
+ current_time = Time.zone.now
19
+
20
+ if (current_time - last_logged_time) >= log_interval
21
+ # Use number_to_human_size for formatting
22
+ readable_retrieved = number_to_human_size(retrieved)
23
+ readable_total = number_to_human_size(total)
24
+ Rails.logger.info "Downloaded #{readable_retrieved} of #{readable_total}, #{filename}: #{percentage.round}% complete"
25
+ last_logged_time = current_time
26
+ end
163
27
  end
28
+ Rails.logger.info "Download complete: #{file['url']} to #{target_file}"
164
29
  end
165
30
  end
166
31
  end
@@ -26,7 +26,7 @@ module Bulkrax
26
26
  end
27
27
  # rubocop:enable Rails/SkipsModelValidations
28
28
  end
29
- return entry if exporter_run.reload.enqueued_records.positive?
29
+ return entry if exporter_run.enqueued_records.positive?
30
30
 
31
31
  if exporter_run.failed_records.positive?
32
32
  exporter_run.exporter.set_status_info('Complete (with failures)')
@@ -2,7 +2,7 @@
2
2
 
3
3
  module Bulkrax
4
4
  class Status < ApplicationRecord
5
- belongs_to :statusable, polymorphic: true, denormalize: { fields: %i[status_message error_class], if: :latest? }
5
+ belongs_to :statusable, polymorphic: true, denormalize: { fields: %i[status_message], if: :latest? }
6
6
  belongs_to :runnable, polymorphic: true
7
7
  serialize :error_backtrace, Array
8
8
 
@@ -26,15 +26,11 @@ module Bulkrax
26
26
 
27
27
  # Prepend the file_set id to ensure a unique filename and also one that is not longer than 255 characters
28
28
  def filename(file_set)
29
+ # NOTE: Will this work with Valkyrie?
29
30
  return if file_set.original_file.blank?
30
- if file_set.original_file.respond_to?(:original_filename) # valkyrie
31
- fn = file_set.original_file.original_filename
32
- mime = ::Marcel::MimeType.for(file_set.original_file.file.io)
33
- else # original non valkyrie version
34
- fn = file_set.original_file.file_name.first
35
- mime = ::Marcel::MimeType.for(declared_type: file_set.original_file.mime_type)
36
- end
37
- ext_mime = ::Marcel::MimeType.for(name: fn)
31
+ fn = file_set.original_file.file_name.first
32
+ mime = ::Marcel::MimeType.for(file_set.original_file.mime_type)
33
+ ext_mime = ::Marcel::MimeType.for(file_set.original_file.file_name)
38
34
  if fn.include?(file_set.id) || importerexporter.metadata_only?
39
35
  filename = "#{fn}.#{mime.to_sym}"
40
36
  filename = fn if mime.to_s == ext_mime.to_s
@@ -430,38 +430,15 @@ module Bulkrax
430
430
  def unzip(file_to_unzip)
431
431
  return untar(file_to_unzip) if file_to_unzip.end_with?('.tar.gz')
432
432
 
433
- dest_dir = importer_unzip_path
434
433
  Zip::File.open(file_to_unzip) do |zip_file|
435
434
  zip_file.each do |entry|
436
- # Zip Slip: reject/confine via the same Bulkrax::SafePath used
437
- # for cloud-file destinations, rather than a bare File.join of
438
- # attacker-controlled entry.name.
439
- entry_path = Bulkrax::SafePath.confine!(dest_dir, entry.name)
435
+ entry_path = File.join(importer_unzip_path, entry.name)
440
436
  FileUtils.mkdir_p(File.dirname(entry_path))
441
- next if File.exist?(entry_path)
442
-
443
- extract_zip_entry(zip_file, entry, dest_dir, entry.name, entry_path)
437
+ zip_file.extract(entry, entry_path) unless File.exist?(entry_path)
444
438
  end
445
439
  end
446
440
  end
447
441
 
448
- # rubyzip 2.x: extract(entry, absolute_dest_path)
449
- # rubyzip 3.x: extract(entry, relative_name, destination_directory: dir)
450
- #
451
- # entry_path (from Bulkrax::SafePath.confine!) is always an absolute
452
- # path; rubyzip 3.x's 2-positional-arg #extract treats its second
453
- # argument as a name relative to the current directory rather than a
454
- # destination path, which silently re-joins an already-absolute path
455
- # onto the cwd. The keyword form is the only one both versions can be
456
- # trusted to interpret as "this absolute/relative destination".
457
- def extract_zip_entry(zip_file, entry, dest_dir, relative_name, dest_path)
458
- if zip_file.method(:extract).arity == 2
459
- zip_file.extract(entry, dest_path)
460
- else
461
- zip_file.extract(entry, relative_name, destination_directory: dest_dir)
462
- end
463
- end
464
-
465
442
  def untar(file_to_untar)
466
443
  Dir.mkdir(importer_unzip_path) unless File.directory?(importer_unzip_path)
467
444
  command = "tar -xzf #{Shellwords.escape(file_to_untar)} -C #{Shellwords.escape(importer_unzip_path)}"
@@ -166,19 +166,12 @@ unless ENV.fetch('BULKRAX_NO_BAGIT', 'false').to_s == 'true'
166
166
  # @todo - investigate getting directory structure
167
167
  # @todo - investigate using perform_later, and having the importer check for
168
168
  # DownloadCloudFileJob before it starts
169
- def retrieve_cloud_files(files, importer)
169
+ def retrieve_cloud_files(files, _importer)
170
170
  # There should only be one zip file for Bagit, take the first
171
171
  return if files['0'].blank?
172
-
173
- file = files['0']
174
- # Checked on the raw file_name before any normalization — see
175
- # Bulkrax::CsvParser#plan_cloud_file for why this must come first.
176
- Bulkrax::CloudSourceValidator.preflight!(file)
177
-
178
- relative_filename = file['file_name'].tr(' ', '_')
179
- target_file = Bulkrax::SafePath.confine!(path_for_import, relative_filename, require_file: true)
172
+ target_file = File.join(path_for_import, files['0']['file_name'].tr(' ', '_'))
180
173
  # Now because we want the files in place before the importer runs
181
- Bulkrax::DownloadCloudFileJob.perform_now(file, importer_id: importer.id, relative_filename: relative_filename)
174
+ Bulkrax::DownloadCloudFileJob.perform_now(files['0'], target_file)
182
175
  return target_file
183
176
  end
184
177
 
@@ -194,55 +194,24 @@ module Bulkrax
194
194
  # @todo - investigate using perform_later, and having the importer check for
195
195
  # DownloadCloudFileJob before it starts
196
196
  def retrieve_cloud_files(files, importer)
197
- plan = plan_cloud_files(files)
198
- dispatch_cloud_files!(plan, importer)
199
- importer[:parser_fields]['original_file_paths'] = plan.map { |_file, _relative_filename, target_file| target_file }
200
- return nil
201
- end
202
-
203
- # Validates and resolves the whole batch, without enqueueing anything
204
- # yet. Otherwise an earlier, valid file's job would already be running
205
- # by the time a later file in the same batch is found to be unsafe.
206
- # Split out from {#retrieve_cloud_files} so a caller (the importers
207
- # controller) can persist the resolved paths before dispatching.
208
- #
209
- # @return [Array<Array(Hash, String, String)>] one entry per file: the
210
- # file spec, its relative destination filename, and the resolved
211
- # absolute target path
212
- def plan_cloud_files(files)
213
197
  files_path = File.join(path_for_import, 'files')
214
198
  FileUtils.mkdir_p(files_path) unless File.exist?(files_path)
215
-
216
- files.values.map { |file| plan_cloud_file(file) }
217
- end
218
-
219
- # Enqueues a download job for each entry of an already-validated plan
220
- # (see {#plan_cloud_files}).
221
- def dispatch_cloud_files!(plan, importer)
222
- plan.each { |file, relative_filename, _target_file| Bulkrax::DownloadCloudFileJob.perform_later(file, importer_id: importer.id, relative_filename: relative_filename) }
223
- end
224
-
225
- # Validates one cloud-file spec and resolves its destination, without
226
- # enqueueing anything yet.
227
- #
228
- # @return [Array(Hash, String, String)] the file spec, its relative
229
- # destination filename, and the resolved absolute target path
230
- def plan_cloud_file(file)
231
- # fixes bug where auth headers do not get attached properly
232
- if file['auth_header'].present?
233
- file['headers'] ||= {}
234
- file['headers'].merge!(file['auth_header'])
199
+ target_files = []
200
+ files.each_pair do |_key, file|
201
+ # fixes bug where auth headers do not get attached properly
202
+ if file['auth_header'].present?
203
+ file['headers'] ||= {}
204
+ file['headers'].merge!(file['auth_header'])
205
+ end
206
+ # this only works for uniquely named files
207
+ target_file = File.join(files_path, file['file_name'].tr(' ', '_'))
208
+ target_files << target_file
209
+ # Now because we want the files in place before the importer runs
210
+ # Problematic for a large upload
211
+ Bulkrax::DownloadCloudFileJob.perform_later(file, target_file)
235
212
  end
236
-
237
- # Checked on the raw file_name, before it's joined under 'files/':
238
- # that join would otherwise neutralize a leading `/` (File.join
239
- # doesn't treat it as absolute once it's not the first argument).
240
- Bulkrax::CloudSourceValidator.preflight!(file)
241
-
242
- # this only works for uniquely named files
243
- relative_filename = File.join('files', file['file_name'].tr(' ', '_'))
244
- target_file = Bulkrax::SafePath.confine!(path_for_import, relative_filename, require_file: true)
245
- [file, relative_filename, target_file]
213
+ importer[:parser_fields]['original_file_paths'] = target_files
214
+ return nil
246
215
  end
247
216
 
248
217
  # export methods
@@ -275,10 +244,7 @@ module Bulkrax
275
244
  record = Bulkrax.object_factory.find(identifier)
276
245
  return unless record
277
246
 
278
- file_sets = Array.wrap(record) if record.file_set?
279
- if file_sets.nil? # for valkyrie
280
- file_sets = record.respond_to?(:file_sets) ? record.file_sets : record.members&.select(&:file_set?)
281
- end
247
+ file_sets = record.file_set? ? Array.wrap(record) : record.file_sets
282
248
  file_sets << record.thumbnail if exporter.include_thumbnails && record.thumbnail.present? && record.work?
283
249
  file_sets.each do |fs|
284
250
  path = File.join(exporter_export_path, folder_count, 'files')
@@ -286,7 +252,7 @@ module Bulkrax
286
252
  file = filename(fs)
287
253
  next if file.blank? || fs.original_file.blank?
288
254
 
289
- io = fs.original_file.respond_to?(:uri) ? open(fs.original_file.uri) : fs.original_file.file.io
255
+ io = open(fs.original_file.uri)
290
256
  File.open(File.join(path, file), 'wb') do |f|
291
257
  f.write(io.read)
292
258
  f.close
@@ -173,7 +173,7 @@ module Bulkrax
173
173
  # @see https://github.com/samvera/hyrax/blob/64c0bbf0dc0d3e1b49f040b50ea70d177cc9d8f6/app/indexers/hyrax/work_indexer.rb#L15-L18
174
174
  def file_sets
175
175
  @file_sets ||= ParserExportRecordSet.in_batches(candidate_file_set_ids) do |batch_of_ids|
176
- fsq = "has_model_ssim:\"#{Bulkrax.file_model_internal_resource.demodulize}\" AND id:(\"" + batch_of_ids.join('" OR "') + "\")"
176
+ fsq = "has_model_ssim:#{Bulkrax.file_model_internal_resource} AND id:(\"" + batch_of_ids.join('" OR "') + "\")"
177
177
  fsq += extra_filters if extra_filters.present?
178
178
  Bulkrax.object_factory.query(
179
179
  fsq,
@@ -41,9 +41,16 @@
41
41
  <p>File upload and Cloud File upload must be a Zip file containing a single BagIt Bag, or a folder containing multiple BagIt Bags.</p>
42
42
  <p>The Server Path can point to a BagIt Bag, a folder containing BagIt Bags, or a zip file containing either.</p>
43
43
 
44
- <%= fi.input :file_style, collection: ['Upload a File', 'Specify a Path on the Server', 'Add Cloud File'], as: :radio_buttons, label: false %>
44
+ <%= fi.input :file_style,
45
+ collection: ['Upload a File', 'Specify a Path on the Server'] +
46
+ (defined?(::Hyrax) && Hyrax.config.browse_everything? ? ['Add Cloud File'] : []),
47
+ as: :radio_buttons, label: false %>
45
48
  <div id='file_upload'>
46
- <%= fi.input 'file', as: :file, input_html: {accept: 'application/zip'} %><br />
49
+ <% if defined?(::Hyrax) %>
50
+ <%= render 'bulkrax/importers/file_uploader', accepted_file_types: 'application/zip' %>
51
+ <% else %>
52
+ <%= fi.input 'file', as: :file, input_html: {accept: 'application/zip'} %><br />
53
+ <% end %>
47
54
  </div>
48
55
  <div id='file_path'>
49
56
  <%= fi.input :import_file_path, as: :string, input_html: { value: importer.parser_fields['import_file_path'] } %>
@@ -53,4 +60,4 @@
53
60
  <%= render 'browse_everything', form: form %>
54
61
  <% end %>
55
62
  </div>
56
- </div>
63
+ </div>
@@ -1,5 +1,4 @@
1
1
  <div class='csv_fields'>
2
-
3
2
  <%= fi.input :visibility,
4
3
  label: 'Default Visibility',
5
4
  collection: [
@@ -11,7 +10,6 @@
11
10
  input_html: { class: 'form-control' },
12
11
  hint: 'If your CSV includes the visibility field, it will override the default setting.'
13
12
  %>
14
-
15
13
  <% if defined?(::Hyrax) %>
16
14
  <% rights_statements = Hyrax.config.rights_statement_service_class.new %>
17
15
  <%= fi.input :rights_statement,
@@ -24,26 +22,32 @@
24
22
  %>
25
23
  <%= fi.input :override_rights_statement, as: :boolean, hint: 'If checked, always use the selected rights statment. If unchecked, use rights or rights_statement from the record and only use the provided value if dc:rights is blank.', input_html: { checked: (importer.parser_fields['override_rights_statement'] == "1") } %>
26
24
  <% end %>
27
- <h4>Add CSV File to Import:</h4>
25
+ <h4>Add CSV or ZIP File to Import:</h4>
28
26
  <%# accept a single file upload; data files and bags will need to be added another way %>
29
-
30
27
  <% file_style_list = ['Upload a File', 'Specify a Path on the Server'] %>
31
28
  <% file_style_list << 'Existing Entries' unless importer.new_record? %>
32
29
  <%= fi.input :file_style, collection: file_style_list, as: :radio_buttons, label: false %>
30
+
33
31
  <div id='file_upload'>
34
- <%= fi.input 'file', as: :file, input_html: { accept: 'text/csv,application/zip,application/gzip' } %><br />
32
+ <% if defined?(::Hyrax) %>
33
+ <%= render 'bulkrax/importers/file_uploader', accepted_file_types: 'text/csv,application/zip,application/gzip' %>
34
+ <% else %>
35
+ <%= fi.input 'file', as: :file, input_html: { accept: 'text/csv,application/zip,application/gzip' } %><br />
36
+ <% end %>
35
37
  </div>
38
+
36
39
  <div id='file_path'>
37
40
  <%= fi.input :import_file_path, as: :string, input_html: { value: importer.parser_fields['import_file_path'] } %>
38
41
  </div>
42
+
39
43
  <div id='existing_options'>
40
44
  <%= fi.collection_check_boxes :entry_statuses, [['Failed'], ['Pending'], ['Skipped'], ['Deleted'], ['Complete']], :first, :first %>
41
45
  </div>
42
-
46
+
43
47
  <% if defined?(::Hyrax) && Hyrax.config.browse_everything? %>
44
- <h4>Add Files to Import:</h4>
45
- <p>Choose files to upload. The filenames must be unique, and the filenames must be referenced in a column called 'file' in the accompanying CSV file.</p>
48
+ <h4>Add Files to Import:</h4>
49
+ <p>Choose files to upload. The filenames must be unique, and the filenames must be referenced in a column called 'file' in the accompanying CSV file.</p>
46
50
  <%= render 'browse_everything', form: form %>
47
51
  <% end %>
48
52
  <br />
49
- </div>
53
+ </div>
@@ -0,0 +1,37 @@
1
+ <div class="fileupload-bulkrax">
2
+ <noscript><input type="hidden" name="redirect" value="<%= main_app.root_path %>" /></noscript>
3
+ <table role="presentation" class="table table-striped"><tbody class="files"></tbody></table>
4
+ <div class="fileupload-buttonbar">
5
+ <div class="row">
6
+ <div class="col-12">
7
+ <div class="fileinput-button" id="add-files">
8
+ <input id="addfiles" type="file" style="display:none;" name="files[]"
9
+ accept="<%= accepted_file_types || 'text/csv,application/zip,application/gzip' %>" multiple />
10
+ <button type="button" class="btn btn-success" onclick="document.getElementById('addfiles').click();">
11
+ <span class="fa fa-plus"></span>
12
+ <span>Add Files</span>
13
+ </button>
14
+ </div>
15
+ <button type="reset" id="file-upload-cancel-btn" class="btn btn-warning cancel">
16
+ <span class="fa fa-ban"></span>
17
+ <span>Cancel Upload</span>
18
+ </button>
19
+ <span class="fileupload-process"></span>
20
+ </div>
21
+ </div>
22
+ <div class="row">
23
+ <div class="col-12">
24
+ <div class="fileupload-progress fade">
25
+ <div class="progress" role="progressbar" aria-valuemin="0" aria-valuemax="100">
26
+ <div class="progress-bar progress-bar-striped progress-bar-animated bg-success" style="width:0%;"></div>
27
+ </div>
28
+ <div class="progress-extended">&nbsp;</div>
29
+ </div>
30
+ </div>
31
+ </div>
32
+ </div>
33
+ <div class="dropzone">
34
+ Drop files here to upload
35
+ </div>
36
+ <%= render 'hyrax/uploads/js_templates' %>
37
+ </div>
@@ -47,16 +47,22 @@
47
47
  <p>File upload and Cloud File upload MUST be a either a single XML file (for metadata only import) OR a Zip file containing the XML files and data files, each in a separate folder.</p>
48
48
  <p>The Server Path can point to a folder containing XML files and data files to import, or direct to the XML file itself.</p>
49
49
 
50
- <%= fi.input :file_style, collection: ['Upload a File', 'Specify a Path on the Server', 'Add Cloud File'], as: :radio_buttons, label: false %>
50
+ <%= fi.input :file_style,
51
+ collection: ['Upload a File', 'Specify a Path on the Server'] +
52
+ (defined?(::Hyrax) && Hyrax.config.browse_everything? ? ['Add Cloud File'] : []),
53
+ as: :radio_buttons, label: false %>
51
54
  <div id='file_upload'>
52
- <%= fi.input 'file', as: :file, input_html: {accept: ['application/zip', 'application/xml']} %><br />
55
+ <% if defined?(::Hyrax) %>
56
+ <%= render 'bulkrax/importers/file_uploader', accepted_file_types: 'application/zip,application/xml' %>
57
+ <% else %>
58
+ <%= fi.input 'file', as: :file, input_html: {accept: ['application/zip', 'application/xml']} %><br />
59
+ <% end%>
53
60
  </div>
54
61
  <div id='file_path'>
55
62
  <%= fi.input :import_file_path, as: :string, input_html: { value: importer.parser_fields['import_file_path'] } %>
56
63
  </div>
57
64
  <div id='cloud'>
58
65
  <% if defined?(::Hyrax) && Hyrax.config.browse_everything? %>
59
- <%= render 'browse_everything', form: form %>
60
66
  <% end %>
61
67
  </div>
62
- </div>
68
+ </div>
@@ -70,9 +70,6 @@ en:
70
70
  type: Type
71
71
  user: User
72
72
  work_entries: Work Entries
73
- browse_everything:
74
- errors:
75
- missing_destination_context: This cloud file download is missing required destination information and can no longer be safely processed. Please re-select the file and try the import again.
76
73
  table_header:
77
74
  labels:
78
75
  identifier: Identifier
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Bulkrax
4
- VERSION = '8.2.4'
4
+ VERSION = '8.3.0'
5
5
  end
data/lib/bulkrax.rb CHANGED
@@ -13,7 +13,6 @@ require 'language_list'
13
13
  require 'marcel'
14
14
  require 'nokogiri'
15
15
  require 'ostruct'
16
- require 'ssrf_filter'
17
16
  require 'zip'
18
17
 
19
18
  def conditional_require(gem_name)
@@ -9,7 +9,7 @@ namespace :bulkrax do
9
9
  progress_mark: ' ',
10
10
  remainder_mark: "\u{FF65}")
11
11
  Bulkrax::Status.latest_by_statusable.includes(:statusable).find_each do |status|
12
- status.statusable.update(status_message: status.status_message, error_class: status.error_class)
12
+ status.statusable.update(status_message: status.status_message)
13
13
  @progress.increment
14
14
  end
15
15
  end