bulkrax 2.1.0 → 2.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/models/bulkrax/csv_file_set_entry.rb +1 -1
- data/app/models/concerns/bulkrax/download_behavior.rb +1 -1
- data/app/parsers/bulkrax/csv_parser.rb +52 -18
- data/app/views/bulkrax/exporters/index.html.erb +4 -0
- data/app/views/bulkrax/importers/index.html.erb +4 -0
- data/lib/bulkrax/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5c5379ae0706c7c37e870551d1da67a526d793b4dc91e9afd41fe275d39c4d48
|
4
|
+
data.tar.gz: 2de4caa074846ebacdfcaed54265011a8a1b8f9d97309e6aa219a4e36497d6e7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e2f138b62a46cc199f7d538294e7850fa3e2bbe9bfa4f3b2643dde3c354b0366d39d08222577a339955f9446257f09deaaa889d4d172c809f230ec2cb48309c0
|
7
|
+
data.tar.gz: 8ca99cfc04657975c93b4e8379df1672c0254a6cd5612bd5bff69b9fdfcf3b98381fbd757b2a85531150f3ae02e149f660c23f41133a63640a4e254719fbc7e8
|
@@ -12,7 +12,7 @@ module Bulkrax
|
|
12
12
|
|
13
13
|
parsed_metadata['file'][i] = path_to_file
|
14
14
|
end
|
15
|
-
raise ::StandardError,
|
15
|
+
raise ::StandardError, "one or more file paths are invalid: #{parsed_metadata['file'].join(', ')}" unless parsed_metadata['file'].map { |file_path| ::File.file?(file_path) }.all?
|
16
16
|
|
17
17
|
parsed_metadata['file']
|
18
18
|
end
|
@@ -30,7 +30,7 @@ module Bulkrax
|
|
30
30
|
|
31
31
|
# Create some headers for the datastream
|
32
32
|
def content_options
|
33
|
-
{ disposition: '
|
33
|
+
{ disposition: 'attachment', type: download_content_type, filename: file_name }
|
34
34
|
end
|
35
35
|
|
36
36
|
# render an HTTP HEAD response
|
@@ -99,7 +99,7 @@ module Bulkrax
|
|
99
99
|
metadata = if collection.delete(:from_collection_field_mapping)
|
100
100
|
uci = unique_collection_identifier(collection)
|
101
101
|
{
|
102
|
-
title:
|
102
|
+
title: collection[:title],
|
103
103
|
work_identifier => uci,
|
104
104
|
source_identifier => uci,
|
105
105
|
visibility: 'open',
|
@@ -110,9 +110,9 @@ module Bulkrax
|
|
110
110
|
## END
|
111
111
|
|
112
112
|
new_entry = find_or_create_entry(collection_entry_class, collection_hash[source_identifier], 'Bulkrax::Importer', collection_hash)
|
113
|
+
increment_counters(index, collection: true)
|
113
114
|
# TODO: add support for :delete option
|
114
115
|
ImportCollectionJob.perform_now(new_entry.id, current_run.id)
|
115
|
-
increment_counters(index, collection: true)
|
116
116
|
end
|
117
117
|
importer.record_status
|
118
118
|
rescue StandardError => e
|
@@ -177,40 +177,74 @@ module Bulkrax
|
|
177
177
|
end
|
178
178
|
|
179
179
|
def current_work_ids
|
180
|
+
ActiveSupport::Deprication.warn('Bulkrax::CsvParser#current_work_ids will be replaced with #current_record_ids in version 3.0')
|
181
|
+
current_record_ids
|
182
|
+
end
|
183
|
+
|
184
|
+
def current_record_ids
|
185
|
+
@work_ids = []
|
186
|
+
@collection_ids = []
|
187
|
+
@file_set_ids = []
|
188
|
+
|
180
189
|
case importerexporter.export_from
|
181
190
|
when 'all'
|
182
|
-
ActiveFedora::SolrService.query("has_model_ssim:(#{Hyrax.config.curation_concerns.join(' OR ')}) #{extra_filters}", rows: 2_147_483_647).map(&:id)
|
191
|
+
@work_ids = ActiveFedora::SolrService.query("has_model_ssim:(#{Hyrax.config.curation_concerns.join(' OR ')}) #{extra_filters}", rows: 2_147_483_647).map(&:id)
|
192
|
+
@collection_ids = ActiveFedora::SolrService.query("has_model_ssim:Collection #{extra_filters}", rows: 2_147_483_647).map(&:id)
|
193
|
+
@file_set_ids = ActiveFedora::SolrService.query("has_model_ssim:FileSet #{extra_filters}", rows: 2_147_483_647).map(&:id)
|
183
194
|
when 'collection'
|
184
|
-
ActiveFedora::SolrService.query("member_of_collection_ids_ssim:#{importerexporter.export_source + extra_filters}", rows: 2_000_000_000).map(&:id)
|
195
|
+
@work_ids = ActiveFedora::SolrService.query("member_of_collection_ids_ssim:#{importerexporter.export_source + extra_filters}", rows: 2_000_000_000).map(&:id)
|
185
196
|
when 'worktype'
|
186
|
-
ActiveFedora::SolrService.query("has_model_ssim:#{importerexporter.export_source + extra_filters}", rows: 2_000_000_000).map(&:id)
|
197
|
+
@work_ids = ActiveFedora::SolrService.query("has_model_ssim:#{importerexporter.export_source + extra_filters}", rows: 2_000_000_000).map(&:id)
|
187
198
|
when 'importer'
|
188
|
-
|
189
|
-
|
190
|
-
.includes(:statusable)
|
191
|
-
.where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', entry_ids, 'Bulkrax::Entry', 'Complete')
|
199
|
+
set_ids_for_exporting_from_importer
|
200
|
+
end
|
192
201
|
|
193
|
-
|
194
|
-
|
202
|
+
@work_ids + @collection_ids + @file_set_ids
|
203
|
+
end
|
204
|
+
|
205
|
+
# Set the following instance variables: @work_ids, @collection_ids, @file_set_ids
|
206
|
+
# @see #current_record_ids
|
207
|
+
def set_ids_for_exporting_from_importer
|
208
|
+
entry_ids = Importer.find(importerexporter.export_source).entries.pluck(:id)
|
209
|
+
complete_statuses = Status.latest_by_statusable
|
210
|
+
.includes(:statusable)
|
211
|
+
.where('bulkrax_statuses.statusable_id IN (?) AND bulkrax_statuses.statusable_type = ? AND status_message = ?', entry_ids, 'Bulkrax::Entry', 'Complete')
|
212
|
+
|
213
|
+
complete_entry_identifiers = complete_statuses.map { |s| s.statusable&.identifier&.gsub(':', '\:') }
|
214
|
+
extra_filters = extra_filters.presence || '*:*'
|
195
215
|
|
196
|
-
|
216
|
+
{ :@work_ids => ::Hyrax.config.curation_concerns, :@collection_ids => [::Collection], :@file_set_ids => [::FileSet] }.each do |instance_var, models_to_search|
|
217
|
+
instance_variable_set(instance_var, ActiveFedora::SolrService.get(
|
197
218
|
extra_filters.to_s,
|
198
|
-
fq:
|
219
|
+
fq: [
|
220
|
+
"#{work_identifier}_sim:(#{complete_entry_identifiers.join(' OR ')})",
|
221
|
+
"has_model_ssim:(#{models_to_search.join(' OR ')})"
|
222
|
+
],
|
199
223
|
fl: 'id',
|
200
224
|
rows: 2_000_000_000
|
201
|
-
)['response']['docs'].map { |obj| obj['id'] }
|
225
|
+
)['response']['docs'].map { |obj| obj['id'] })
|
202
226
|
end
|
203
227
|
end
|
204
228
|
|
205
229
|
def create_new_entries
|
206
|
-
|
230
|
+
current_record_ids.each_with_index do |id, index|
|
207
231
|
break if limit_reached?(limit, index)
|
208
|
-
|
232
|
+
|
233
|
+
this_entry_class = if @collection_ids.include?(id)
|
234
|
+
collection_entry_class
|
235
|
+
elsif @file_set_ids.include?(id)
|
236
|
+
file_set_entry_class
|
237
|
+
else
|
238
|
+
entry_class
|
239
|
+
end
|
240
|
+
new_entry = find_or_create_entry(this_entry_class, id, 'Bulkrax::Exporter')
|
241
|
+
|
209
242
|
begin
|
210
|
-
entry =
|
243
|
+
entry = ExportWorkJob.perform_now(new_entry.id, current_run.id)
|
211
244
|
rescue => e
|
212
245
|
Rails.logger.info("#{e.message} was detected during export")
|
213
246
|
end
|
247
|
+
|
214
248
|
self.headers |= entry.parsed_metadata.keys if entry
|
215
249
|
end
|
216
250
|
end
|
@@ -267,7 +301,7 @@ module Bulkrax
|
|
267
301
|
|
268
302
|
def write_files
|
269
303
|
CSV.open(setup_export_file, "w", headers: export_headers, write_headers: true) do |csv|
|
270
|
-
importerexporter.entries.where(identifier:
|
304
|
+
importerexporter.entries.where(identifier: current_record_ids)[0..limit || total].each do |e|
|
271
305
|
csv << e.parsed_metadata
|
272
306
|
end
|
273
307
|
end
|
data/lib/bulkrax/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bulkrax
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rob Kaufman
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-02-
|
11
|
+
date: 2022-02-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|