bulkrax 4.3.1 → 4.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/assets/javascripts/bulkrax/exporters.js +12 -0
- data/app/controllers/bulkrax/importers_controller.rb +1 -1
- data/app/factories/bulkrax/object_factory.rb +35 -9
- data/app/jobs/bulkrax/create_relationships_job.rb +1 -1
- data/app/jobs/bulkrax/import_work_job.rb +12 -10
- data/app/matchers/bulkrax/application_matcher.rb +1 -1
- data/app/models/bulkrax/csv_entry.rb +5 -5
- data/app/models/bulkrax/importer.rb +20 -15
- data/app/models/bulkrax/oai_entry.rb +1 -2
- data/app/models/concerns/bulkrax/file_set_entry_behavior.rb +8 -1
- data/app/models/concerns/bulkrax/import_behavior.rb +10 -9
- data/app/parsers/bulkrax/application_parser.rb +84 -13
- data/app/parsers/bulkrax/csv_parser.rb +3 -3
- data/app/parsers/bulkrax/oai_dc_parser.rb +2 -2
- data/app/services/bulkrax/remove_relationships_for_importer.rb +107 -0
- data/app/views/bulkrax/exporters/_form.html.erb +3 -3
- data/app/views/bulkrax/shared/_file_set_entries_tab.html.erb +3 -3
- data/app/views/hyrax/dashboard/sidebar/_bulkrax_sidebar_additions.html.erb +7 -5
- data/app/views/hyrax/dashboard/sidebar/_repository_content.html.erb +23 -15
- data/db/migrate/20211203195233_rename_children_counters_to_relationships.rb +1 -1
- data/db/migrate/20211220195027_add_file_set_counters_to_importer_runs.rb +1 -1
- data/db/migrate/20220118001339_add_import_attempts_to_entries.rb +1 -1
- data/db/migrate/20220119213325_add_work_counters_to_importer_runs.rb +1 -1
- data/db/migrate/20220301001839_create_bulkrax_pending_relationships.rb +1 -1
- data/db/migrate/20220303212810_add_order_to_bulkrax_pending_relationships.rb +1 -1
- data/db/migrate/20220412233954_add_include_thumbnails_to_bulkrax_exporters.rb +1 -1
- data/db/migrate/20220413180915_add_generated_metadata_to_bulkrax_exporters.rb +1 -1
- data/db/migrate/20220609001128_rename_bulkrax_importer_run_to_importer_run.rb +1 -1
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +38 -11
- data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb +10 -0
- data/lib/tasks/reset.rake +65 -0
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d53012e0252f8033f5da334d5336c134deb3a6221040cfe4c1c6c01bb473d617
|
4
|
+
data.tar.gz: e986a3506c073aa533c4ea303a123e8866d2bf7ac6d8ab6df3949682f61b6c05
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8ea20593d1164f62fdfda2dbe84bd8cc4e199a65d7e85eea0b68b27e71aeff7579f1b0059042f55eec1c7ab36f8dec9db406ad58cc3a315b29b8e4900f7cb450
|
7
|
+
data.tar.gz: 89a03cd842d855d48f8c4952b6e9a6daea757e80cb8c0f9ee370b59f90e434b39dfb548deb1757f36463914bd5cccbcf1df6c5ffd9102bb5f0d54b5b7ea6476f
|
@@ -1,10 +1,12 @@
|
|
1
1
|
function hideUnhide(field) {
|
2
2
|
var allSources = $('body').find('.export-source-option')
|
3
|
+
removeRequired(allSources)
|
3
4
|
hide(allSources)
|
4
5
|
|
5
6
|
if (field.length > 0) {
|
6
7
|
var selectedSource = $('.' + field)
|
7
8
|
unhideSelected(selectedSource)
|
9
|
+
addRequired(selectedSource)
|
8
10
|
}
|
9
11
|
|
10
12
|
if (field === 'collection') {
|
@@ -12,6 +14,16 @@ function hideUnhide(field) {
|
|
12
14
|
}
|
13
15
|
};
|
14
16
|
|
17
|
+
function addRequired(selectedSource) {
|
18
|
+
selectedSource.addClass('required').attr('required', 'required');
|
19
|
+
selectedSource.parent().addClass('required');
|
20
|
+
}
|
21
|
+
|
22
|
+
function removeRequired(allSources) {
|
23
|
+
allSources.removeClass('required').removeAttr('required');
|
24
|
+
allSources.parent().removeClass('required').removeAttr('required')
|
25
|
+
};
|
26
|
+
|
15
27
|
// hide all export_source
|
16
28
|
function hide(allSources) {
|
17
29
|
allSources.addClass('hidden');
|
@@ -276,7 +276,7 @@ module Bulkrax
|
|
276
276
|
def setup_client(url)
|
277
277
|
return false if url.nil?
|
278
278
|
headers = { from: Bulkrax.server_name }
|
279
|
-
@client ||= OAI::Client.new(url, headers: headers, parser: 'libxml'
|
279
|
+
@client ||= OAI::Client.new(url, headers: headers, parser: 'libxml')
|
280
280
|
end
|
281
281
|
|
282
282
|
# Download methods
|
@@ -159,28 +159,55 @@ module Bulkrax
|
|
159
159
|
file_set_attrs = attrs.slice(*object.attributes.keys)
|
160
160
|
object.assign_attributes(file_set_attrs)
|
161
161
|
|
162
|
-
attrs['uploaded_files']
|
162
|
+
attrs['uploaded_files']&.each do |uploaded_file_id|
|
163
163
|
uploaded_file = ::Hyrax::UploadedFile.find(uploaded_file_id)
|
164
164
|
next if uploaded_file.file_set_uri.present?
|
165
165
|
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
actor.create_content(uploaded_file)
|
171
|
-
actor.attach_to_work(work)
|
166
|
+
create_file_set_actor(attrs, work, work_permissions, uploaded_file)
|
167
|
+
end
|
168
|
+
attrs['remote_files']&.each do |remote_file|
|
169
|
+
create_file_set_actor(attrs, work, work_permissions, nil, remote_file)
|
172
170
|
end
|
173
171
|
|
174
172
|
object.save!
|
175
173
|
end
|
176
174
|
|
175
|
+
def create_file_set_actor(attrs, work, work_permissions, uploaded_file, remote_file = nil)
|
176
|
+
actor = ::Hyrax::Actors::FileSetActor.new(object, @user)
|
177
|
+
uploaded_file&.update(file_set_uri: actor.file_set.uri)
|
178
|
+
actor.file_set.permissions_attributes = work_permissions
|
179
|
+
actor.create_metadata(attrs)
|
180
|
+
actor.create_content(uploaded_file) if uploaded_file
|
181
|
+
actor.attach_to_work(work, attrs)
|
182
|
+
handle_remote_file(remote_file: remote_file, actor: actor, update: false) if remote_file
|
183
|
+
end
|
184
|
+
|
177
185
|
def update_file_set(attrs)
|
178
186
|
file_set_attrs = attrs.slice(*object.attributes.keys)
|
179
187
|
actor = ::Hyrax::Actors::FileSetActor.new(object, @user)
|
180
|
-
|
188
|
+
attrs['remote_files']&.each do |remote_file|
|
189
|
+
handle_remote_file(remote_file: remote_file, actor: actor, update: true)
|
190
|
+
end
|
181
191
|
actor.update_metadata(file_set_attrs)
|
182
192
|
end
|
183
193
|
|
194
|
+
def handle_remote_file(remote_file:, actor:, update: false)
|
195
|
+
actor.file_set.label = remote_file['file_name']
|
196
|
+
actor.file_set.import_url = remote_file['url']
|
197
|
+
|
198
|
+
url = remote_file['url']
|
199
|
+
tmp_file = Tempfile.new(remote_file['file_name'].split('.').first)
|
200
|
+
tmp_file.binmode
|
201
|
+
|
202
|
+
URI.open(url) do |url_file|
|
203
|
+
tmp_file.write(url_file.read)
|
204
|
+
end
|
205
|
+
|
206
|
+
tmp_file.rewind
|
207
|
+
update == true ? actor.update_content(tmp_file) : actor.create_content(tmp_file, from_url: true)
|
208
|
+
tmp_file.close
|
209
|
+
end
|
210
|
+
|
184
211
|
def clean_attrs(attrs)
|
185
212
|
# avoid the "ArgumentError: Identifier must be a string of size > 0 in order to be treeified" error
|
186
213
|
# when setting object.attributes
|
@@ -200,7 +227,6 @@ module Bulkrax
|
|
200
227
|
def transform_attributes(update: false)
|
201
228
|
@transform_attributes = attributes.slice(*permitted_attributes)
|
202
229
|
@transform_attributes.merge!(file_attributes(update_files)) if with_files
|
203
|
-
@transform_attributes.transform_values! { |v| v == [""] ? [] : v }
|
204
230
|
update ? @transform_attributes.except(:id) : @transform_attributes
|
205
231
|
end
|
206
232
|
|
@@ -81,7 +81,7 @@ module Bulkrax
|
|
81
81
|
# This is adding the reverse relationship, from the child to the parent
|
82
82
|
def collection_parent_work_child
|
83
83
|
child_work_ids = child_records[:works].map(&:id)
|
84
|
-
parent_record.reindex_extent
|
84
|
+
parent_record.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX)
|
85
85
|
|
86
86
|
parent_record.add_member_objects(child_work_ids)
|
87
87
|
ImporterRun.find(importer_run_id).increment!(:processed_relationships, child_work_ids.count) # rubocop:disable Rails/SkipsModelValidations
|
@@ -5,25 +5,27 @@ module Bulkrax
|
|
5
5
|
queue_as :import
|
6
6
|
|
7
7
|
# rubocop:disable Rails/SkipsModelValidations
|
8
|
-
def perform(*
|
9
|
-
entry = Entry.find(
|
8
|
+
def perform(entry_id, run_id, *)
|
9
|
+
entry = Entry.find(entry_id)
|
10
|
+
importer_run = ImporterRun.find(run_id)
|
10
11
|
entry.build
|
11
12
|
if entry.status == "Complete"
|
12
|
-
|
13
|
-
|
14
|
-
ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
|
13
|
+
importer_run.increment!(:processed_records)
|
14
|
+
importer_run.increment!(:processed_works)
|
15
15
|
else
|
16
16
|
# do not retry here because whatever parse error kept you from creating a work will likely
|
17
17
|
# keep preventing you from doing so.
|
18
|
-
|
19
|
-
|
20
|
-
ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
|
18
|
+
importer_run.increment!(:failed_records)
|
19
|
+
importer_run.increment!(:failed_works)
|
21
20
|
end
|
21
|
+
# Regardless of completion or not, we want to decrement the enqueued records.
|
22
|
+
importer_run.decrement!(:enqueued_records) unless importer_run.enqueued_records <= 0
|
23
|
+
|
22
24
|
entry.save!
|
23
|
-
entry.importer.current_run =
|
25
|
+
entry.importer.current_run = importer_run
|
24
26
|
entry.importer.record_status
|
25
27
|
rescue Bulkrax::CollectionsCreatedError
|
26
|
-
reschedule(
|
28
|
+
reschedule(entry_id, run_id)
|
27
29
|
end
|
28
30
|
# rubocop:enable Rails/SkipsModelValidations
|
29
31
|
|
@@ -30,7 +30,7 @@ module Bulkrax
|
|
30
30
|
|
31
31
|
def process_split
|
32
32
|
if self.split.is_a?(TrueClass)
|
33
|
-
@result = @result.split(
|
33
|
+
@result = @result.split(Bulkrax.multi_value_element_split_on)
|
34
34
|
elsif self.split
|
35
35
|
result = @result.split(Regexp.new(self.split))
|
36
36
|
@result = result.map(&:strip)
|
@@ -18,7 +18,7 @@ module Bulkrax
|
|
18
18
|
raise StandardError, 'CSV path empty' if path.blank?
|
19
19
|
CSV.read(path,
|
20
20
|
headers: true,
|
21
|
-
header_converters:
|
21
|
+
header_converters: ->(h) { h.to_sym },
|
22
22
|
encoding: 'utf-8')
|
23
23
|
end
|
24
24
|
|
@@ -81,7 +81,7 @@ module Bulkrax
|
|
81
81
|
def add_file
|
82
82
|
self.parsed_metadata['file'] ||= []
|
83
83
|
if record['file']&.is_a?(String)
|
84
|
-
self.parsed_metadata['file'] = record['file'].split(
|
84
|
+
self.parsed_metadata['file'] = record['file'].split(Bulkrax.multi_value_element_split_on)
|
85
85
|
elsif record['file'].is_a?(Array)
|
86
86
|
self.parsed_metadata['file'] = record['file']
|
87
87
|
end
|
@@ -176,7 +176,7 @@ module Bulkrax
|
|
176
176
|
data = hyrax_record.send(key.to_s)
|
177
177
|
if data.is_a?(ActiveTriples::Relation)
|
178
178
|
if value['join']
|
179
|
-
self.parsed_metadata[key_for_export(key)] = data.map { |d| prepare_export_data(d) }.join(
|
179
|
+
self.parsed_metadata[key_for_export(key)] = data.map { |d| prepare_export_data(d) }.join(Bulkrax.multi_value_element_join_on).to_s
|
180
180
|
else
|
181
181
|
data.each_with_index do |d, i|
|
182
182
|
self.parsed_metadata["#{key_for_export(key)}_#{i + 1}"] = prepare_export_data(d)
|
@@ -236,7 +236,7 @@ module Bulkrax
|
|
236
236
|
|
237
237
|
def handle_join_on_export(key, values, join)
|
238
238
|
if join
|
239
|
-
parsed_metadata[key] = values.join(
|
239
|
+
parsed_metadata[key] = values.join(Bulkrax.multi_value_element_join_on)
|
240
240
|
else
|
241
241
|
values.each_with_index do |value, i|
|
242
242
|
parsed_metadata["#{key}_#{i + 1}"] = value
|
@@ -260,7 +260,7 @@ module Bulkrax
|
|
260
260
|
return [] unless parent_field_mapping.present? && record[parent_field_mapping].present?
|
261
261
|
|
262
262
|
identifiers = []
|
263
|
-
split_references = record[parent_field_mapping].split(
|
263
|
+
split_references = record[parent_field_mapping].split(Bulkrax.multi_value_element_split_on)
|
264
264
|
split_references.each do |c_reference|
|
265
265
|
matching_collection_entries = importerexporter.entries.select do |e|
|
266
266
|
(e.raw_metadata&.[](source_identifier) == c_reference) &&
|
@@ -58,17 +58,26 @@ module Bulkrax
|
|
58
58
|
|
59
59
|
# If field_mapping is empty, setup a default based on the export_properties
|
60
60
|
def mapping
|
61
|
+
# rubocop:disable Style/IfUnlessModifier
|
61
62
|
@mapping ||= if self.field_mapping.blank? || self.field_mapping == [{}]
|
62
63
|
if parser.import_fields.present? || self.field_mapping == [{}]
|
63
|
-
|
64
|
-
parser.import_fields.reject(&:nil?).map do |m|
|
65
|
-
Bulkrax.default_field_mapping.call(m)
|
66
|
-
end.inject(:merge)
|
67
|
-
)
|
64
|
+
default_field_mapping
|
68
65
|
end
|
69
66
|
else
|
70
|
-
self.field_mapping
|
67
|
+
default_field_mapping.merge(self.field_mapping)
|
71
68
|
end
|
69
|
+
|
70
|
+
# rubocop:enable Style/IfUnlessModifier
|
71
|
+
end
|
72
|
+
|
73
|
+
def default_field_mapping
|
74
|
+
return self.field_mapping if parser.import_fields.nil?
|
75
|
+
|
76
|
+
ActiveSupport::HashWithIndifferentAccess.new(
|
77
|
+
parser.import_fields.reject(&:nil?).map do |m|
|
78
|
+
Bulkrax.default_field_mapping.call(m)
|
79
|
+
end.inject(:merge)
|
80
|
+
)
|
72
81
|
end
|
73
82
|
|
74
83
|
def parser_fields
|
@@ -143,17 +152,13 @@ module Bulkrax
|
|
143
152
|
import_objects(['relationship'])
|
144
153
|
end
|
145
154
|
|
155
|
+
DEFAULT_OBJECT_TYPES = %w[collection work file_set relationship].freeze
|
156
|
+
|
146
157
|
def import_objects(types_array = nil)
|
147
158
|
self.only_updates ||= false
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
else
|
152
|
-
types.each do |object_type|
|
153
|
-
self.save if self.new_record? # Object needs to be saved for statuses
|
154
|
-
parser.send("create_#{object_type.pluralize}")
|
155
|
-
end
|
156
|
-
end
|
159
|
+
self.save if self.new_record? # Object needs to be saved for statuses
|
160
|
+
types = types_array || DEFAULT_OBJECT_TYPES
|
161
|
+
parser.create_objects(types)
|
157
162
|
rescue StandardError => e
|
158
163
|
status_info(e)
|
159
164
|
end
|
@@ -59,12 +59,11 @@ module Bulkrax
|
|
59
59
|
def find_collection_ids
|
60
60
|
return self.collection_ids if collections_created?
|
61
61
|
if sets.blank? || parser.collection_name != 'all'
|
62
|
-
# c = Collection.where(Bulkrax.system_identifier_field => importerexporter.unique_collection_identifier(parser.collection_name)).first
|
63
62
|
collection = find_collection(importerexporter.unique_collection_identifier(parser.collection_name))
|
64
63
|
self.collection_ids << collection.id if collection.present? && !self.collection_ids.include?(collection.id)
|
65
64
|
else # All - collections should exist for all sets
|
66
65
|
sets.each do |set|
|
67
|
-
c =
|
66
|
+
c = find_collection(importerexporter.unique_collection_identifier(set.content))
|
68
67
|
self.collection_ids << c.id if c.present? && !self.collection_ids.include?(c.id)
|
69
68
|
end
|
70
69
|
end
|
@@ -6,7 +6,14 @@ module Bulkrax
|
|
6
6
|
::FileSet
|
7
7
|
end
|
8
8
|
|
9
|
+
def file_reference
|
10
|
+
return 'file' if parsed_metadata&.[]('file')&.map(&:present?)&.any?
|
11
|
+
return 'remote_files' if parsed_metadata&.[]('remote_files')&.map(&:present?)&.any?
|
12
|
+
end
|
13
|
+
|
9
14
|
def add_path_to_file
|
15
|
+
return unless file_reference == 'file'
|
16
|
+
|
10
17
|
parsed_metadata['file'].each_with_index do |filename, i|
|
11
18
|
next if filename.blank?
|
12
19
|
|
@@ -22,7 +29,7 @@ module Bulkrax
|
|
22
29
|
end
|
23
30
|
|
24
31
|
def validate_presence_of_filename!
|
25
|
-
return if parsed_metadata&.[](
|
32
|
+
return if parsed_metadata&.[](file_reference)&.map(&:present?)&.any?
|
26
33
|
|
27
34
|
raise StandardError, 'File set must have a filename'
|
28
35
|
end
|
@@ -165,15 +165,16 @@ module Bulkrax
|
|
165
165
|
end
|
166
166
|
|
167
167
|
def factory
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
168
|
+
of = Bulkrax.object_factory || Bulkrax::ObjectFactory
|
169
|
+
@factory ||= of.new(attributes: self.parsed_metadata,
|
170
|
+
source_identifier_value: identifier,
|
171
|
+
work_identifier: parser.work_identifier,
|
172
|
+
related_parents_parsed_mapping: parser.related_parents_parsed_mapping,
|
173
|
+
replace_files: replace_files,
|
174
|
+
user: user,
|
175
|
+
klass: factory_class,
|
176
|
+
importer_run_id: importerexporter.last_run.id,
|
177
|
+
update_files: update_files)
|
177
178
|
end
|
178
179
|
|
179
180
|
def factory_class
|
@@ -2,6 +2,9 @@
|
|
2
2
|
require 'zip'
|
3
3
|
|
4
4
|
module Bulkrax
|
5
|
+
# An abstract class that establishes the API for Bulkrax's import and export parsing.
|
6
|
+
#
|
7
|
+
# @abstract Subclass the Bulkrax::ApplicationParser to create a parser that handles a specific format (e.g. CSV, Bagit, XML, etc).
|
5
8
|
class ApplicationParser # rubocop:disable Metrics/ClassLength
|
6
9
|
attr_accessor :importerexporter, :headers
|
7
10
|
alias importer importerexporter
|
@@ -12,14 +15,21 @@ module Bulkrax
|
|
12
15
|
:exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
|
13
16
|
to: :importerexporter
|
14
17
|
|
18
|
+
# @todo Convert to `class_attribute :parser_fiels, default: {}`
|
15
19
|
def self.parser_fields
|
16
20
|
{}
|
17
21
|
end
|
18
22
|
|
23
|
+
# @return [TrueClass,FalseClass] this parser does or does not support exports.
|
24
|
+
#
|
25
|
+
# @todo Convert to `class_attribute :export_supported, default: false, instance_predicate: true` and `self << class; alias export_supported? export_supported; end`
|
19
26
|
def self.export_supported?
|
20
27
|
false
|
21
28
|
end
|
22
29
|
|
30
|
+
# @return [TrueClass,FalseClass] this parser does or does not support imports.
|
31
|
+
#
|
32
|
+
# @todo Convert to `class_attribute :import_supported, default: false, instance_predicate: true` and `self << class; alias import_supported? import_supported; end`
|
23
33
|
def self.import_supported?
|
24
34
|
true
|
25
35
|
end
|
@@ -29,49 +39,70 @@ module Bulkrax
|
|
29
39
|
@headers = []
|
30
40
|
end
|
31
41
|
|
32
|
-
# @api
|
42
|
+
# @api public
|
43
|
+
# @abstract Subclass and override {#entry_class} to implement behavior for the parser.
|
33
44
|
def entry_class
|
34
|
-
raise
|
45
|
+
raise NotImplementedError, 'must be defined'
|
35
46
|
end
|
36
47
|
|
37
|
-
# @api
|
48
|
+
# @api public
|
49
|
+
# @abstract Subclass and override {#collection_entry_class} to implement behavior for the parser.
|
38
50
|
def collection_entry_class
|
39
|
-
raise
|
51
|
+
raise NotImplementedError, 'must be defined'
|
40
52
|
end
|
41
53
|
|
42
|
-
# @api
|
54
|
+
# @api public
|
55
|
+
# @abstract Subclass and override {#records} to implement behavior for the parser.
|
43
56
|
def records(_opts = {})
|
44
|
-
raise
|
57
|
+
raise NotImplementedError, 'must be defined'
|
45
58
|
end
|
46
59
|
|
60
|
+
# @return [Symbol] the name of the identifying property in the source system from which we're
|
61
|
+
# importing (e.g. is *not* this application that mounts *this* Bulkrax engine).
|
62
|
+
#
|
63
|
+
# @see #work_identifier
|
64
|
+
# @see https://github.com/samvera-labs/bulkrax/wiki/CSV-Importer#source-identifier Bulkrax Wiki regarding source identifier
|
47
65
|
def source_identifier
|
48
66
|
@source_identifier ||= get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('from')&.first&.to_sym || :source_identifier
|
49
67
|
end
|
50
68
|
|
69
|
+
# @return [Symbol] the name of the identifying property for the system which we're importing
|
70
|
+
# into (e.g. the application that mounts *this* Bulkrax engine)
|
71
|
+
# @see #source_identifier
|
51
72
|
def work_identifier
|
52
73
|
@work_identifier ||= get_field_mapping_hash_for('source_identifier')&.keys&.first&.to_sym || :source
|
53
74
|
end
|
54
75
|
|
76
|
+
# @return [String]
|
55
77
|
def generated_metadata_mapping
|
56
78
|
@generated_metadata_mapping ||= 'generated'
|
57
79
|
end
|
58
80
|
|
81
|
+
# @return [String, NilClass]
|
82
|
+
# @see #related_parents_raw_mapping
|
59
83
|
def related_parents_raw_mapping
|
60
84
|
@related_parents_raw_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.values&.first&.[]('from')&.first
|
61
85
|
end
|
62
86
|
|
87
|
+
# @return [String]
|
88
|
+
# @see #related_parents_field_mapping
|
63
89
|
def related_parents_parsed_mapping
|
64
90
|
@related_parents_parsed_mapping ||= (get_field_mapping_hash_for('related_parents_field_mapping')&.keys&.first || 'parents')
|
65
91
|
end
|
66
92
|
|
93
|
+
# @return [String, NilClass]
|
94
|
+
# @see #related_children_parsed_mapping
|
67
95
|
def related_children_raw_mapping
|
68
96
|
@related_children_raw_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.values&.first&.[]('from')&.first
|
69
97
|
end
|
70
98
|
|
99
|
+
# @return [String]
|
100
|
+
# @see #related_children_raw_mapping
|
71
101
|
def related_children_parsed_mapping
|
72
102
|
@related_children_parsed_mapping ||= (get_field_mapping_hash_for('related_children_field_mapping')&.keys&.first || 'children')
|
73
103
|
end
|
74
104
|
|
105
|
+
# @api private
|
75
106
|
def get_field_mapping_hash_for(key)
|
76
107
|
return instance_variable_get("@#{key}_hash") if instance_variable_get("@#{key}_hash").present?
|
77
108
|
|
@@ -85,6 +116,7 @@ module Bulkrax
|
|
85
116
|
instance_variable_get("@#{key}_hash")
|
86
117
|
end
|
87
118
|
|
119
|
+
# @return [Array<String>]
|
88
120
|
def model_field_mappings
|
89
121
|
model_mappings = Bulkrax.field_mappings[self.class.to_s]&.dig('model', :from) || []
|
90
122
|
model_mappings |= ['model']
|
@@ -92,6 +124,7 @@ module Bulkrax
|
|
92
124
|
model_mappings
|
93
125
|
end
|
94
126
|
|
127
|
+
# @return [String]
|
95
128
|
def perform_method
|
96
129
|
if self.validate_only
|
97
130
|
'perform_now'
|
@@ -100,29 +133,55 @@ module Bulkrax
|
|
100
133
|
end
|
101
134
|
end
|
102
135
|
|
136
|
+
# The visibility of the record. Acceptable values are: "open", "embaro", "lease", "authenticated", "restricted". The default is "open"
|
137
|
+
#
|
138
|
+
# @return [String]
|
139
|
+
# @see https://github.com/samvera/hydra-head/blob/main/hydra-access-controls/app/models/concerns/hydra/access_controls/access_right.rb Hydra::AccessControls::AccessRight for details on the range of values.
|
140
|
+
# @see https://github.com/samvera/hyrax/blob/bd2bcffc33e183904be2c175367648815f25bc2b/app/services/hyrax/visibility_intention.rb Hyrax::VisibilityIntention for how we process the visibility.
|
103
141
|
def visibility
|
104
142
|
@visibility ||= self.parser_fields['visibility'] || 'open'
|
105
143
|
end
|
106
144
|
|
145
|
+
# @api public
|
146
|
+
#
|
147
|
+
# @param types [Array<Symbol>] the types of objects that we'll create.
|
148
|
+
#
|
149
|
+
# @see Bulkrax::Importer::DEFAULT_OBJECT_TYPES
|
150
|
+
# @see #create_collections
|
151
|
+
# @see #create_works
|
152
|
+
# @see #create_file_sets
|
153
|
+
# @see #create_relationships
|
154
|
+
def create_objects(types = [])
|
155
|
+
types.each do |object_type|
|
156
|
+
parser.send("create_#{object_type.pluralize}")
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
# @abstract Subclass and override {#create_collections} to implement behavior for the parser.
|
107
161
|
def create_collections
|
108
|
-
raise
|
162
|
+
raise NotImplementedError, 'must be defined' if importer?
|
109
163
|
end
|
110
164
|
|
165
|
+
# @abstract Subclass and override {#create_works} to implement behavior for the parser.
|
111
166
|
def create_works
|
112
|
-
raise
|
167
|
+
raise NotImplementedError, 'must be defined' if importer?
|
113
168
|
end
|
114
169
|
|
170
|
+
# @abstract Subclass and override {#create_file_sets} to implement behavior for the parser.
|
115
171
|
def create_file_sets
|
116
|
-
raise
|
172
|
+
raise NotImplementedError, 'must be defined' if importer?
|
117
173
|
end
|
118
174
|
|
175
|
+
# @abstract Subclass and override {#create_relationships} to implement behavior for the parser.
|
119
176
|
def create_relationships
|
120
|
-
raise
|
177
|
+
raise NotImplementedError, 'must be defined' if importer?
|
121
178
|
end
|
122
179
|
|
123
180
|
# Optional, define if using browse everything for file upload
|
124
181
|
def retrieve_cloud_files(files); end
|
125
182
|
|
183
|
+
# @param file [#path, #original_filename] the file object that with the relevant data for the
|
184
|
+
# import.
|
126
185
|
def write_import_file(file)
|
127
186
|
path = File.join(path_for_import, file.original_filename)
|
128
187
|
FileUtils.mv(
|
@@ -133,6 +192,8 @@ module Bulkrax
|
|
133
192
|
end
|
134
193
|
|
135
194
|
# Base path for imported and exported files
|
195
|
+
# @param [String]
|
196
|
+
# @return [String] the base path for files that this parser will "parse"
|
136
197
|
def base_path(type = 'import')
|
137
198
|
# account for multiple versions of hyku
|
138
199
|
is_multitenant = ENV['HYKU_MULTITENANT'] == 'true' || ENV['SETTINGS__MULTITENANCY__ENABLED'] == 'true'
|
@@ -141,41 +202,48 @@ module Bulkrax
|
|
141
202
|
|
142
203
|
# Path where we'll store the import metadata and files
|
143
204
|
# this is used for uploaded and cloud files
|
205
|
+
# @return [String]
|
144
206
|
def path_for_import
|
145
207
|
@path_for_import = File.join(base_path, importerexporter.path_string)
|
146
208
|
FileUtils.mkdir_p(@path_for_import) unless File.exist?(@path_for_import)
|
147
209
|
@path_for_import
|
148
210
|
end
|
149
211
|
|
212
|
+
# @abstract Subclass and override {#setup_export_file} to implement behavior for the parser.
|
150
213
|
def setup_export_file
|
151
|
-
raise
|
214
|
+
raise NotImplementedError, 'must be defined' if exporter?
|
152
215
|
end
|
153
216
|
|
217
|
+
# @abstract Subclass and override {#write_files} to implement behavior for the parser.
|
154
218
|
def write_files
|
155
|
-
raise
|
219
|
+
raise NotImplementedError, 'must be defined' if exporter?
|
156
220
|
end
|
157
221
|
|
222
|
+
# @return [TrueClass,FalseClass]
|
158
223
|
def importer?
|
159
224
|
importerexporter.is_a?(Bulkrax::Importer)
|
160
225
|
end
|
161
226
|
|
227
|
+
# @return [TrueClass,FalseClass]
|
162
228
|
def exporter?
|
163
229
|
importerexporter.is_a?(Bulkrax::Exporter)
|
164
230
|
end
|
165
231
|
|
166
232
|
# @param limit [Integer] limit set on the importerexporter
|
167
233
|
# @param index [Integer] index of current iteration
|
168
|
-
# @return [
|
234
|
+
# @return [TrueClass,FalseClass]
|
169
235
|
def limit_reached?(limit, index)
|
170
236
|
return false if limit.nil? || limit.zero? # no limit
|
171
237
|
index >= limit
|
172
238
|
end
|
173
239
|
|
174
240
|
# Override to add specific validations
|
241
|
+
# @return [TrueClass,FalseClass]
|
175
242
|
def valid_import?
|
176
243
|
true
|
177
244
|
end
|
178
245
|
|
246
|
+
# @return [TrueClass,FalseClass]
|
179
247
|
def record_has_source_identifier(record, index)
|
180
248
|
if record[source_identifier].blank?
|
181
249
|
if Bulkrax.fill_in_blank_source_identifiers.present?
|
@@ -199,6 +267,7 @@ module Bulkrax
|
|
199
267
|
end
|
200
268
|
# rubocop:enable Rails/SkipsModelValidations
|
201
269
|
|
270
|
+
# @return [Array<String>]
|
202
271
|
def required_elements
|
203
272
|
if Bulkrax.fill_in_blank_source_identifiers
|
204
273
|
['title']
|
@@ -287,12 +356,14 @@ module Bulkrax
|
|
287
356
|
end
|
288
357
|
|
289
358
|
# Path for the import
|
359
|
+
# @return [String]
|
290
360
|
def import_file_path
|
291
361
|
@import_file_path ||= real_import_file_path
|
292
362
|
end
|
293
363
|
|
294
364
|
private
|
295
365
|
|
366
|
+
# @return [String]
|
296
367
|
def real_import_file_path
|
297
368
|
return importer_unzip_path if file? && zip?
|
298
369
|
parser_fields['import_file_path']
|
@@ -180,7 +180,7 @@ module Bulkrax
|
|
180
180
|
end
|
181
181
|
|
182
182
|
def current_work_ids
|
183
|
-
ActiveSupport::
|
183
|
+
ActiveSupport::Deprecation.warn('Bulkrax::CsvParser#current_work_ids will be replaced with #current_record_ids in version 3.0')
|
184
184
|
current_record_ids
|
185
185
|
end
|
186
186
|
|
@@ -440,7 +440,7 @@ module Bulkrax
|
|
440
440
|
file_mapping = Bulkrax.field_mappings.dig(self.class.to_s, 'file', :from)&.first&.to_sym || :file
|
441
441
|
next if r[file_mapping].blank?
|
442
442
|
|
443
|
-
r[file_mapping].split(
|
443
|
+
r[file_mapping].split(Bulkrax.multi_value_element_split_on).map do |f|
|
444
444
|
file = File.join(path_to_files, f.tr(' ', '_'))
|
445
445
|
if File.exist?(file) # rubocop:disable Style/GuardClause
|
446
446
|
file
|
@@ -468,7 +468,7 @@ module Bulkrax
|
|
468
468
|
entry_uid ||= if Bulkrax.fill_in_blank_source_identifiers.present?
|
469
469
|
Bulkrax.fill_in_blank_source_identifiers.call(self, records.find_index(collection_hash))
|
470
470
|
else
|
471
|
-
collection_hash[:title].split(
|
471
|
+
collection_hash[:title].split(Bulkrax.multi_value_element_split_on).first
|
472
472
|
end
|
473
473
|
|
474
474
|
entry_uid
|
@@ -13,8 +13,7 @@ module Bulkrax
|
|
13
13
|
def client
|
14
14
|
@client ||= OAI::Client.new(importerexporter.parser_fields['base_url'],
|
15
15
|
headers: headers,
|
16
|
-
parser: 'libxml'
|
17
|
-
metadata_prefix: importerexporter.parser_fields['metadata_prefix'])
|
16
|
+
parser: 'libxml')
|
18
17
|
rescue StandardError
|
19
18
|
raise OAIError
|
20
19
|
end
|
@@ -32,6 +31,7 @@ module Bulkrax
|
|
32
31
|
end
|
33
32
|
|
34
33
|
def records(opts = {})
|
34
|
+
opts[:metadata_prefix] ||= importerexporter.parser_fields['metadata_prefix']
|
35
35
|
opts[:set] = collection_name unless collection_name == 'all'
|
36
36
|
|
37
37
|
opts[:from] = importerexporter&.last_imported_at&.strftime("%Y-%m-%d") if importerexporter.last_imported_at && only_updates
|
@@ -0,0 +1,107 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Bulkrax
|
3
|
+
# This module is rather destructive; it will break relationships between the works, file sets, and
|
4
|
+
# collections that were imported via an importer. You probably don't want to run this on your
|
5
|
+
# data, except in the case where you have been testing a Bulkrax::Importer, the parsers and
|
6
|
+
# mappings. Then, you might have relationships that you want to remove.
|
7
|
+
#
|
8
|
+
# tl;dr - Caution this will break things!
|
9
|
+
class RemoveRelationshipsForImporter
|
10
|
+
# @api public
|
11
|
+
#
|
12
|
+
# Remove the relationships of the works and collections for all of the Bulkrax::Entry records
|
13
|
+
# associated with the given Bulkrax::Importer.
|
14
|
+
#
|
15
|
+
# @param importer [Bulkrax::Importer]
|
16
|
+
# @param with_progress_bar [Boolean]
|
17
|
+
def self.break_relationships_for!(importer:, with_progress_bar: false)
|
18
|
+
entries = importer.entries.select(&:succeeded?)
|
19
|
+
progress_bar = build_progress_bar_for(with_progress_bar: with_progress_bar, entries: entries)
|
20
|
+
new(progress_bar: progress_bar, entries: entries).break_relationships!
|
21
|
+
end
|
22
|
+
|
23
|
+
# @api private
|
24
|
+
#
|
25
|
+
# A null object that conforms to this class's use of a progress bar.
|
26
|
+
module NullProgressBar
|
27
|
+
def self.increment; end
|
28
|
+
end
|
29
|
+
|
30
|
+
# @api private
|
31
|
+
#
|
32
|
+
# @return [#increment]
|
33
|
+
def self.build_progress_bar_for(with_progress_bar:, entries:)
|
34
|
+
return NullProgressBar unless with_progress_bar
|
35
|
+
|
36
|
+
begin
|
37
|
+
require 'ruby-progressbar'
|
38
|
+
ProgessBar.create(total: entries.count)
|
39
|
+
rescue LoadError
|
40
|
+
Rails.logger.info("Using NullProgressBar because ProgressBar is not available due to a LoadError.")
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
# @param entries [#each]
|
45
|
+
# @param progress_bar [#increment]
|
46
|
+
def initialize(entries:, progress_bar:)
|
47
|
+
@progress_bar = progress_bar
|
48
|
+
@entries = entries
|
49
|
+
end
|
50
|
+
|
51
|
+
attr_reader :entries, :progress_bar
|
52
|
+
|
53
|
+
def break_relationships!
|
54
|
+
entries.each do |entry|
|
55
|
+
progress_bar.increment
|
56
|
+
|
57
|
+
obj = entry.factory.find
|
58
|
+
next if obj.is_a?(FileSet) # FileSets must be attached to a Work
|
59
|
+
|
60
|
+
if obj.is_a?(Collection)
|
61
|
+
remove_relationships_from_collection(obj)
|
62
|
+
else
|
63
|
+
remove_relationships_from_work(obj)
|
64
|
+
end
|
65
|
+
|
66
|
+
obj.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX)
|
67
|
+
obj.save!
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def remove_relationships_from_collection(collection)
|
72
|
+
# Remove child work relationships
|
73
|
+
collection.member_works.each do |work|
|
74
|
+
change = work.member_of_collections.delete(collection)
|
75
|
+
work.save! if change.present?
|
76
|
+
end
|
77
|
+
|
78
|
+
# Remove parent collection relationships
|
79
|
+
collection.member_of_collections.each do |parent_col|
|
80
|
+
Hyrax::Collections::NestedCollectionPersistenceService
|
81
|
+
.remove_nested_relationship_for(parent: parent_col, child: collection)
|
82
|
+
end
|
83
|
+
|
84
|
+
# Remove child collection relationships
|
85
|
+
collection.member_collections.each do |child_col|
|
86
|
+
Hyrax::Collections::NestedCollectionPersistenceService
|
87
|
+
.remove_nested_relationship_for(parent: collection, child: child_col)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def remove_relationships_from_work(work)
|
92
|
+
# Remove parent collection relationships
|
93
|
+
work.member_of_collections = []
|
94
|
+
|
95
|
+
# Remove parent work relationships
|
96
|
+
work.member_of_works.each do |parent_work|
|
97
|
+
parent_work.members.delete(work)
|
98
|
+
parent_work.save!
|
99
|
+
end
|
100
|
+
|
101
|
+
# Remove child work relationships
|
102
|
+
work.member_works.each do |child_work|
|
103
|
+
work.member_works.delete(child_work)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
@@ -29,7 +29,7 @@
|
|
29
29
|
|
30
30
|
<%= form.input :export_source_importer,
|
31
31
|
label: t('bulkrax.exporter.labels.importer'),
|
32
|
-
|
32
|
+
required: true,
|
33
33
|
prompt: 'Select from the list',
|
34
34
|
label_html: { class: 'importer export-source-option hidden' },
|
35
35
|
input_html: { class: 'importer export-source-option hidden' },
|
@@ -38,7 +38,7 @@
|
|
38
38
|
<%= form.input :export_source_collection,
|
39
39
|
prompt: 'Start typing ...',
|
40
40
|
label: t('bulkrax.exporter.labels.collection'),
|
41
|
-
|
41
|
+
required: true,
|
42
42
|
placeholder: @collection&.title&.first,
|
43
43
|
label_html: { class: 'collection export-source-option hidden' },
|
44
44
|
input_html: {
|
@@ -52,7 +52,7 @@
|
|
52
52
|
|
53
53
|
<%= form.input :export_source_worktype,
|
54
54
|
label: t('bulkrax.exporter.labels.worktype'),
|
55
|
-
|
55
|
+
required: true,
|
56
56
|
prompt: 'Select from the list',
|
57
57
|
label_html: { class: 'worktype export-source-option hidden' },
|
58
58
|
input_html: { class: 'worktype export-source-option hidden' },
|
@@ -14,7 +14,7 @@
|
|
14
14
|
<% entries.each do |e| %>
|
15
15
|
<% entry_path = item.class.to_s.include?('Importer') ? bulkrax.importer_entry_path(item.id, e.id) : bulkrax.exporter_entry_path(item.id, e.id) %>
|
16
16
|
<tr>
|
17
|
-
<td><%= link_to e.identifier,
|
17
|
+
<td><%= link_to e.identifier, entry_path %></td>
|
18
18
|
<td><%= e.id %></td>
|
19
19
|
<% if e.status == "Complete" %>
|
20
20
|
<td><span class="glyphicon glyphicon-ok" style="color: green;"></span> <%= e.status %></td>
|
@@ -24,12 +24,12 @@
|
|
24
24
|
<td><span class="glyphicon glyphicon-remove" style="color: <%= e.status == 'Deleted' ? 'green' : 'red' %>;"></span> <%= e.status %></td>
|
25
25
|
<% end %>
|
26
26
|
<% if e.last_error.present? %>
|
27
|
-
<td><%= link_to e.last_error.dig("error_class"),
|
27
|
+
<td><%= link_to e.last_error.dig("error_class"), entry_path %></td>
|
28
28
|
<% else %>
|
29
29
|
<td></td>
|
30
30
|
<% end %>
|
31
31
|
<td><%= e.status_at %></td>
|
32
|
-
<td><%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'),
|
32
|
+
<td><%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'), entry_path %></td>
|
33
33
|
</tr>
|
34
34
|
<% end %>
|
35
35
|
</tbody>
|
@@ -1,6 +1,8 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
<% if ENV.fetch('HYKU_BULKRAX_ENABLED', 'true') == 'true' %>
|
2
|
+
<%= menu.nav_link(bulkrax.importers_path) do %>
|
3
|
+
<span class="fa fa-cloud-upload" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('bulkrax.admin.sidebar.importers') %></span>
|
4
|
+
<% end %>
|
5
|
+
<%= menu.nav_link(bulkrax.exporters_path) do %>
|
6
|
+
<span class="fa fa-cloud-download" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('bulkrax.admin.sidebar.exporters') %></span>
|
7
|
+
<% end %>
|
3
8
|
<% end %>
|
4
|
-
<%= menu.nav_link(bulkrax.exporters_path) do %>
|
5
|
-
<span class="fa fa-cloud-download" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('bulkrax.admin.sidebar.exporters') %></span>
|
6
|
-
<% end %>
|
@@ -1,19 +1,27 @@
|
|
1
|
-
<li class="h5"><%= t('hyrax.admin.sidebar.repository_objects') %></li>
|
1
|
+
<li class="h5 nav-item"><%= t('hyrax.admin.sidebar.repository_objects') %></li>
|
2
2
|
|
3
|
-
<%= menu.nav_link(hyrax.my_collections_path,
|
4
|
-
|
5
|
-
|
6
|
-
|
3
|
+
<%= menu.nav_link(hyrax.my_collections_path,
|
4
|
+
class: "nav-link",
|
5
|
+
onclick: "dontChangeAccordion(event);",
|
6
|
+
also_active_for: hyrax.dashboard_collections_path) do %>
|
7
|
+
<span class="fa fa-folder-open" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('hyrax.admin.sidebar.collections') %></span>
|
8
|
+
<% end %>
|
7
9
|
|
8
|
-
<%= menu.nav_link(hyrax.my_works_path,
|
9
|
-
|
10
|
-
|
11
|
-
|
10
|
+
<%= menu.nav_link(hyrax.my_works_path,
|
11
|
+
class: "nav-link",
|
12
|
+
onclick: "dontChangeAccordion(event);",
|
13
|
+
also_active_for: hyrax.dashboard_works_path) do %>
|
14
|
+
<span class="fa fa-file" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('hyrax.admin.sidebar.works') %></span>
|
15
|
+
<% end %>
|
12
16
|
|
13
|
-
|
14
|
-
|
15
|
-
<%
|
17
|
+
<% if ::Hyrax::DashboardController&.respond_to?(:sidebar_partials) %>
|
18
|
+
<%= render 'hyrax/dashboard/sidebar/menu_partials', menu: menu, section: :repository_content %>
|
19
|
+
<% else %>
|
20
|
+
<%= menu.nav_link(bulkrax.importers_path) do %>
|
21
|
+
<span class="fa fa-cloud-upload" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('bulkrax.admin.sidebar.importers') %></span>
|
22
|
+
<% end %>
|
16
23
|
|
17
|
-
<%= menu.nav_link(bulkrax.exporters_path) do %>
|
18
|
-
|
19
|
-
<% end %>
|
24
|
+
<%= menu.nav_link(bulkrax.exporters_path) do %>
|
25
|
+
<span class="fa fa-cloud-download" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('bulkrax.admin.sidebar.exporters') %></span>
|
26
|
+
<% end %>
|
27
|
+
<% end %>
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class RenameChildrenCountersToRelationships < ActiveRecord::Migration[5.
|
1
|
+
class RenameChildrenCountersToRelationships < ActiveRecord::Migration[5.1]
|
2
2
|
def change
|
3
3
|
rename_column :bulkrax_importer_runs, :processed_children, :processed_relationships unless column_exists?(:bulkrax_importer_runs, :processed_relationships)
|
4
4
|
rename_column :bulkrax_importer_runs, :failed_children, :failed_relationships unless column_exists?(:bulkrax_importer_runs, :failed_relationships)
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class AddFileSetCountersToImporterRuns < ActiveRecord::Migration[5.
|
1
|
+
class AddFileSetCountersToImporterRuns < ActiveRecord::Migration[5.1]
|
2
2
|
def change
|
3
3
|
add_column :bulkrax_importer_runs, :processed_file_sets, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :processed_file_sets)
|
4
4
|
add_column :bulkrax_importer_runs, :failed_file_sets, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :failed_file_sets)
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class AddImportAttemptsToEntries < ActiveRecord::Migration[5.
|
1
|
+
class AddImportAttemptsToEntries < ActiveRecord::Migration[5.1]
|
2
2
|
def change
|
3
3
|
add_column :bulkrax_entries, :import_attempts, :integer, default: 0 unless column_exists?(:bulkrax_entries, :import_attempts)
|
4
4
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class AddWorkCountersToImporterRuns < ActiveRecord::Migration[5.
|
1
|
+
class AddWorkCountersToImporterRuns < ActiveRecord::Migration[5.1]
|
2
2
|
def change
|
3
3
|
add_column :bulkrax_importer_runs, :processed_works, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :processed_works)
|
4
4
|
add_column :bulkrax_importer_runs, :failed_works, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :failed_works)
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class AddOrderToBulkraxPendingRelationships < ActiveRecord::Migration[5.
|
1
|
+
class AddOrderToBulkraxPendingRelationships < ActiveRecord::Migration[5.1]
|
2
2
|
def change
|
3
3
|
add_column :bulkrax_pending_relationships, :order, :integer, default: 0 unless column_exists?(:bulkrax_pending_relationships, :order)
|
4
4
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class AddIncludeThumbnailsToBulkraxExporters < ActiveRecord::Migration[5.
|
1
|
+
class AddIncludeThumbnailsToBulkraxExporters < ActiveRecord::Migration[5.1]
|
2
2
|
def change
|
3
3
|
add_column :bulkrax_exporters, :include_thumbnails, :boolean, default: false unless column_exists?(:bulkrax_exporters, :include_thumbnails)
|
4
4
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class AddGeneratedMetadataToBulkraxExporters < ActiveRecord::Migration[5.
|
1
|
+
class AddGeneratedMetadataToBulkraxExporters < ActiveRecord::Migration[5.1]
|
2
2
|
def change
|
3
3
|
add_column :bulkrax_exporters, :generated_metadata, :boolean, default: false unless column_exists?(:bulkrax_exporters, :generated_metadata)
|
4
4
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class RenameBulkraxImporterRunToImporterRun < ActiveRecord::Migration[5.
|
1
|
+
class RenameBulkraxImporterRunToImporterRun < ActiveRecord::Migration[5.1]
|
2
2
|
def up
|
3
3
|
if column_exists?(:bulkrax_pending_relationships, :bulkrax_importer_run_id)
|
4
4
|
remove_foreign_key :bulkrax_pending_relationships, :bulkrax_importer_runs
|
data/lib/bulkrax/version.rb
CHANGED
data/lib/bulkrax.rb
CHANGED
@@ -1,25 +1,29 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "bulkrax/version"
|
3
4
|
require "bulkrax/engine"
|
4
5
|
require 'active_support/all'
|
5
6
|
|
6
7
|
module Bulkrax
|
7
8
|
class << self
|
8
|
-
mattr_accessor :
|
9
|
-
:default_work_type,
|
9
|
+
mattr_accessor :api_definition,
|
10
10
|
:default_field_mapping,
|
11
|
+
:default_work_type,
|
12
|
+
:export_path,
|
13
|
+
:field_mappings,
|
11
14
|
:fill_in_blank_source_identifiers,
|
12
15
|
:generated_metadata_mapping,
|
16
|
+
:import_path,
|
17
|
+
:multi_value_element_join_on,
|
18
|
+
:multi_value_element_split_on,
|
19
|
+
:object_factory,
|
20
|
+
:parsers,
|
21
|
+
:qa_controlled_properties,
|
13
22
|
:related_children_field_mapping,
|
14
23
|
:related_parents_field_mapping,
|
15
|
-
:reserved_properties,
|
16
|
-
:qa_controlled_properties,
|
17
|
-
:field_mappings,
|
18
|
-
:import_path,
|
19
|
-
:export_path,
|
20
24
|
:removed_image_path,
|
21
|
-
:
|
22
|
-
:
|
25
|
+
:reserved_properties,
|
26
|
+
:server_name
|
23
27
|
|
24
28
|
self.parsers = [
|
25
29
|
{ name: "OAI - Dublin Core", class_name: "Bulkrax::OaiDcParser", partial: "oai_fields" },
|
@@ -29,8 +33,8 @@ module Bulkrax
|
|
29
33
|
{ name: "XML", class_name: "Bulkrax::XmlParser", partial: "xml_fields" }
|
30
34
|
]
|
31
35
|
|
32
|
-
self.import_path = 'tmp/imports'
|
33
|
-
self.export_path = 'tmp/exports'
|
36
|
+
self.import_path = Bulkrax.import_path || 'tmp/imports'
|
37
|
+
self.export_path = Bulkrax.export_path || 'tmp/exports'
|
34
38
|
self.removed_image_path = Bulkrax::Engine.root.join('spec', 'fixtures', 'removed.png').to_s
|
35
39
|
self.server_name = 'bulkrax@example.com'
|
36
40
|
|
@@ -137,6 +141,29 @@ module Bulkrax
|
|
137
141
|
)
|
138
142
|
end
|
139
143
|
|
144
|
+
DEFAULT_MULTI_VALUE_ELEMENT_JOIN_ON = ' | '
|
145
|
+
# Specify the delimiter for joining an attribute's multi-value array into a string.
|
146
|
+
#
|
147
|
+
# @note the specific delimeter should likely be present in the multi_value_element_split_on
|
148
|
+
# expression.
|
149
|
+
def self.multi_value_element_join_on
|
150
|
+
@multi_value_element_join_on ||= DEFAULT_MULTI_VALUE_ELEMENT_JOIN_ON
|
151
|
+
end
|
152
|
+
|
153
|
+
DEFAULT_MULTI_VALUE_ELEMENT_SPLIT_ON = /\s*[:;|]\s*/.freeze
|
154
|
+
# @return [RegexClass] the regular express to use to "split" an attribute's values. If set to
|
155
|
+
# `true` use the DEFAULT_MULTI_VALUE_ELEMENT_JOIN_ON.
|
156
|
+
#
|
157
|
+
# @note The "true" value is to preserve backwards compatibility.
|
158
|
+
# @see DEFAULT_MULTI_VALUE_ELEMENT_JOIN_ON
|
159
|
+
def self.multi_value_element_split_on
|
160
|
+
if @multi_value_element_join_on.is_a?(TrueClass)
|
161
|
+
DEFAULT_MULTI_VALUE_ELEMENT_SPLIT_ON
|
162
|
+
else
|
163
|
+
@multi_value_element_split_on ||= DEFAULT_MULTI_VALUE_ELEMENT_SPLIT_ON
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
140
167
|
# this function maps the vars from your app into your engine
|
141
168
|
def self.setup
|
142
169
|
yield self
|
@@ -10,6 +10,9 @@ Bulkrax.setup do |config|
|
|
10
10
|
# Default is the first returned by Hyrax.config.curation_concerns
|
11
11
|
# config.default_work_type = MyWork
|
12
12
|
|
13
|
+
# Factory Class to use when generating and saving objects
|
14
|
+
config.object_factory = Bulkrax::ObjectFactory
|
15
|
+
|
13
16
|
# Path to store pending imports
|
14
17
|
# config.import_path = 'tmp/imports'
|
15
18
|
|
@@ -67,6 +70,13 @@ Bulkrax.setup do |config|
|
|
67
70
|
# is controlled by the active terms in config/authorities/rights_statements.yml
|
68
71
|
# Defaults: 'rights_statement' and 'license'
|
69
72
|
# config.qa_controlled_properties += ['my_field']
|
73
|
+
|
74
|
+
# Specify the delimiter regular expression for splitting an attribute's values into a multi-value array.
|
75
|
+
# config.multi_value_element_split_on = //\s*[:;|]\s*/.freeze
|
76
|
+
|
77
|
+
# Specify the delimiter for joining an attribute's multi-value array into a string. Note: the
|
78
|
+
# specific delimeter should likely be present in the multi_value_element_split_on expression.
|
79
|
+
# config.multi_value_element_join_on = ' | '
|
70
80
|
end
|
71
81
|
|
72
82
|
# Sidebar for hyrax 3+ support
|
@@ -0,0 +1,65 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
namespace :hyrax do
|
4
|
+
namespace :reset do
|
5
|
+
desc "Delete Bulkrax importers in addition to the reset below"
|
6
|
+
task importers_et_all: :works_and_collections do
|
7
|
+
# sometimes re-running an existing importer causes issues
|
8
|
+
# and you may need to create new ones or delete the existing ones
|
9
|
+
Bulkrax::Importer.delete_all
|
10
|
+
end
|
11
|
+
|
12
|
+
desc 'Reset fedora / solr and corresponding database tables w/o clearing other active record tables like users'
|
13
|
+
task works_and_collections: [:environment] do
|
14
|
+
confirm('You are about to delete all works and collections, this is not reversable!')
|
15
|
+
require 'active_fedora/cleaner'
|
16
|
+
ActiveFedora::Cleaner.clean!
|
17
|
+
Hyrax::PermissionTemplateAccess.delete_all
|
18
|
+
Hyrax::PermissionTemplate.delete_all
|
19
|
+
Bulkrax::PendingRelationship.delete_all
|
20
|
+
Bulkrax::Entry.delete_all
|
21
|
+
Bulkrax::ImporterRun.delete_all
|
22
|
+
Bulkrax::Status.delete_all
|
23
|
+
# Remove sipity methods, everything but sipity roles
|
24
|
+
Sipity::Workflow.delete_all
|
25
|
+
Sipity::EntitySpecificResponsibility.delete_all
|
26
|
+
Sipity::Comment.delete_all
|
27
|
+
Sipity::Entity.delete_all
|
28
|
+
Sipity::WorkflowRole.delete_all
|
29
|
+
Sipity::WorkflowResponsibility.delete_all
|
30
|
+
Sipity::Agent.delete_all
|
31
|
+
Mailboxer::Receipt.destroy_all
|
32
|
+
Mailboxer::Notification.delete_all
|
33
|
+
Mailboxer::Conversation::OptOut.delete_all
|
34
|
+
Mailboxer::Conversation.delete_all
|
35
|
+
AccountElevator.switch!('single.tenant.default')
|
36
|
+
# we need to wait till Fedora is done with its cleanup
|
37
|
+
# otherwise creating the admin set will fail
|
38
|
+
while AdminSet.exists?(AdminSet::DEFAULT_ID)
|
39
|
+
puts 'waiting for delete to finish before reinitializing Fedora'
|
40
|
+
sleep 20
|
41
|
+
end
|
42
|
+
|
43
|
+
Hyrax::CollectionType.find_or_create_default_collection_type
|
44
|
+
Hyrax::CollectionType.find_or_create_admin_set_type
|
45
|
+
AdminSet.find_or_create_default_admin_set_id
|
46
|
+
|
47
|
+
collection_types = Hyrax::CollectionType.all
|
48
|
+
collection_types.each do |c|
|
49
|
+
next unless c.title.match?(/^translation missing/)
|
50
|
+
oldtitle = c.title
|
51
|
+
c.title = I18n.t(c.title.gsub("translation missing: en.", ''))
|
52
|
+
c.save
|
53
|
+
puts "#{oldtitle} changed to #{c.title}"
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def confirm(action)
|
58
|
+
return if ENV['RESET_CONFIRMED'].present?
|
59
|
+
confirm_token = rand(36**6).to_s(36)
|
60
|
+
STDOUT.puts "#{action} Enter '#{confirm_token}' to confirm:"
|
61
|
+
input = STDIN.gets.chomp
|
62
|
+
raise "Aborting. You entered #{input}" unless input == confirm_token
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bulkrax
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rob Kaufman
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-12-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|
@@ -328,6 +328,7 @@ files:
|
|
328
328
|
- app/parsers/bulkrax/oai_dc_parser.rb
|
329
329
|
- app/parsers/bulkrax/oai_qualified_dc_parser.rb
|
330
330
|
- app/parsers/bulkrax/xml_parser.rb
|
331
|
+
- app/services/bulkrax/remove_relationships_for_importer.rb
|
331
332
|
- app/views/bulkrax/entries/_parsed_metadata.html.erb
|
332
333
|
- app/views/bulkrax/entries/_raw_metadata.html.erb
|
333
334
|
- app/views/bulkrax/entries/show.html.erb
|
@@ -404,6 +405,7 @@ files:
|
|
404
405
|
- lib/generators/bulkrax/templates/config/bulkrax_api.yml
|
405
406
|
- lib/generators/bulkrax/templates/config/initializers/bulkrax.rb
|
406
407
|
- lib/tasks/bulkrax_tasks.rake
|
408
|
+
- lib/tasks/reset.rake
|
407
409
|
homepage: https://github.com/samvera-labs/bulkrax
|
408
410
|
licenses:
|
409
411
|
- Apache-2.0
|
@@ -423,7 +425,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
423
425
|
- !ruby/object:Gem::Version
|
424
426
|
version: '0'
|
425
427
|
requirements: []
|
426
|
-
rubygems_version: 3.
|
428
|
+
rubygems_version: 3.0.3
|
427
429
|
signing_key:
|
428
430
|
specification_version: 4
|
429
431
|
summary: Import and export tool for Hyrax and Hyku
|