bulkrax 4.3.1 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/javascripts/bulkrax/exporters.js +12 -0
- data/app/controllers/bulkrax/importers_controller.rb +1 -1
- data/app/factories/bulkrax/object_factory.rb +35 -9
- data/app/jobs/bulkrax/create_relationships_job.rb +1 -1
- data/app/jobs/bulkrax/import_work_job.rb +12 -10
- data/app/matchers/bulkrax/application_matcher.rb +1 -1
- data/app/models/bulkrax/csv_entry.rb +5 -5
- data/app/models/bulkrax/importer.rb +20 -15
- data/app/models/bulkrax/oai_entry.rb +1 -2
- data/app/models/concerns/bulkrax/file_set_entry_behavior.rb +8 -1
- data/app/models/concerns/bulkrax/import_behavior.rb +10 -9
- data/app/parsers/bulkrax/application_parser.rb +84 -13
- data/app/parsers/bulkrax/csv_parser.rb +3 -3
- data/app/parsers/bulkrax/oai_dc_parser.rb +2 -2
- data/app/services/bulkrax/remove_relationships_for_importer.rb +107 -0
- data/app/views/bulkrax/exporters/_form.html.erb +3 -3
- data/app/views/bulkrax/shared/_file_set_entries_tab.html.erb +3 -3
- data/app/views/hyrax/dashboard/sidebar/_bulkrax_sidebar_additions.html.erb +7 -5
- data/app/views/hyrax/dashboard/sidebar/_repository_content.html.erb +23 -15
- data/db/migrate/20211203195233_rename_children_counters_to_relationships.rb +1 -1
- data/db/migrate/20211220195027_add_file_set_counters_to_importer_runs.rb +1 -1
- data/db/migrate/20220118001339_add_import_attempts_to_entries.rb +1 -1
- data/db/migrate/20220119213325_add_work_counters_to_importer_runs.rb +1 -1
- data/db/migrate/20220301001839_create_bulkrax_pending_relationships.rb +1 -1
- data/db/migrate/20220303212810_add_order_to_bulkrax_pending_relationships.rb +1 -1
- data/db/migrate/20220412233954_add_include_thumbnails_to_bulkrax_exporters.rb +1 -1
- data/db/migrate/20220413180915_add_generated_metadata_to_bulkrax_exporters.rb +1 -1
- data/db/migrate/20220609001128_rename_bulkrax_importer_run_to_importer_run.rb +1 -1
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +38 -11
- data/lib/generators/bulkrax/templates/config/initializers/bulkrax.rb +10 -0
- data/lib/tasks/reset.rake +65 -0
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d53012e0252f8033f5da334d5336c134deb3a6221040cfe4c1c6c01bb473d617
|
4
|
+
data.tar.gz: e986a3506c073aa533c4ea303a123e8866d2bf7ac6d8ab6df3949682f61b6c05
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8ea20593d1164f62fdfda2dbe84bd8cc4e199a65d7e85eea0b68b27e71aeff7579f1b0059042f55eec1c7ab36f8dec9db406ad58cc3a315b29b8e4900f7cb450
|
7
|
+
data.tar.gz: 89a03cd842d855d48f8c4952b6e9a6daea757e80cb8c0f9ee370b59f90e434b39dfb548deb1757f36463914bd5cccbcf1df6c5ffd9102bb5f0d54b5b7ea6476f
|
@@ -1,10 +1,12 @@
|
|
1
1
|
function hideUnhide(field) {
|
2
2
|
var allSources = $('body').find('.export-source-option')
|
3
|
+
removeRequired(allSources)
|
3
4
|
hide(allSources)
|
4
5
|
|
5
6
|
if (field.length > 0) {
|
6
7
|
var selectedSource = $('.' + field)
|
7
8
|
unhideSelected(selectedSource)
|
9
|
+
addRequired(selectedSource)
|
8
10
|
}
|
9
11
|
|
10
12
|
if (field === 'collection') {
|
@@ -12,6 +14,16 @@ function hideUnhide(field) {
|
|
12
14
|
}
|
13
15
|
};
|
14
16
|
|
17
|
+
function addRequired(selectedSource) {
|
18
|
+
selectedSource.addClass('required').attr('required', 'required');
|
19
|
+
selectedSource.parent().addClass('required');
|
20
|
+
}
|
21
|
+
|
22
|
+
function removeRequired(allSources) {
|
23
|
+
allSources.removeClass('required').removeAttr('required');
|
24
|
+
allSources.parent().removeClass('required').removeAttr('required')
|
25
|
+
};
|
26
|
+
|
15
27
|
// hide all export_source
|
16
28
|
function hide(allSources) {
|
17
29
|
allSources.addClass('hidden');
|
@@ -276,7 +276,7 @@ module Bulkrax
|
|
276
276
|
def setup_client(url)
|
277
277
|
return false if url.nil?
|
278
278
|
headers = { from: Bulkrax.server_name }
|
279
|
-
@client ||= OAI::Client.new(url, headers: headers, parser: 'libxml'
|
279
|
+
@client ||= OAI::Client.new(url, headers: headers, parser: 'libxml')
|
280
280
|
end
|
281
281
|
|
282
282
|
# Download methods
|
@@ -159,28 +159,55 @@ module Bulkrax
|
|
159
159
|
file_set_attrs = attrs.slice(*object.attributes.keys)
|
160
160
|
object.assign_attributes(file_set_attrs)
|
161
161
|
|
162
|
-
attrs['uploaded_files']
|
162
|
+
attrs['uploaded_files']&.each do |uploaded_file_id|
|
163
163
|
uploaded_file = ::Hyrax::UploadedFile.find(uploaded_file_id)
|
164
164
|
next if uploaded_file.file_set_uri.present?
|
165
165
|
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
actor.create_content(uploaded_file)
|
171
|
-
actor.attach_to_work(work)
|
166
|
+
create_file_set_actor(attrs, work, work_permissions, uploaded_file)
|
167
|
+
end
|
168
|
+
attrs['remote_files']&.each do |remote_file|
|
169
|
+
create_file_set_actor(attrs, work, work_permissions, nil, remote_file)
|
172
170
|
end
|
173
171
|
|
174
172
|
object.save!
|
175
173
|
end
|
176
174
|
|
175
|
+
def create_file_set_actor(attrs, work, work_permissions, uploaded_file, remote_file = nil)
|
176
|
+
actor = ::Hyrax::Actors::FileSetActor.new(object, @user)
|
177
|
+
uploaded_file&.update(file_set_uri: actor.file_set.uri)
|
178
|
+
actor.file_set.permissions_attributes = work_permissions
|
179
|
+
actor.create_metadata(attrs)
|
180
|
+
actor.create_content(uploaded_file) if uploaded_file
|
181
|
+
actor.attach_to_work(work, attrs)
|
182
|
+
handle_remote_file(remote_file: remote_file, actor: actor, update: false) if remote_file
|
183
|
+
end
|
184
|
+
|
177
185
|
def update_file_set(attrs)
|
178
186
|
file_set_attrs = attrs.slice(*object.attributes.keys)
|
179
187
|
actor = ::Hyrax::Actors::FileSetActor.new(object, @user)
|
180
|
-
|
188
|
+
attrs['remote_files']&.each do |remote_file|
|
189
|
+
handle_remote_file(remote_file: remote_file, actor: actor, update: true)
|
190
|
+
end
|
181
191
|
actor.update_metadata(file_set_attrs)
|
182
192
|
end
|
183
193
|
|
194
|
+
def handle_remote_file(remote_file:, actor:, update: false)
|
195
|
+
actor.file_set.label = remote_file['file_name']
|
196
|
+
actor.file_set.import_url = remote_file['url']
|
197
|
+
|
198
|
+
url = remote_file['url']
|
199
|
+
tmp_file = Tempfile.new(remote_file['file_name'].split('.').first)
|
200
|
+
tmp_file.binmode
|
201
|
+
|
202
|
+
URI.open(url) do |url_file|
|
203
|
+
tmp_file.write(url_file.read)
|
204
|
+
end
|
205
|
+
|
206
|
+
tmp_file.rewind
|
207
|
+
update == true ? actor.update_content(tmp_file) : actor.create_content(tmp_file, from_url: true)
|
208
|
+
tmp_file.close
|
209
|
+
end
|
210
|
+
|
184
211
|
def clean_attrs(attrs)
|
185
212
|
# avoid the "ArgumentError: Identifier must be a string of size > 0 in order to be treeified" error
|
186
213
|
# when setting object.attributes
|
@@ -200,7 +227,6 @@ module Bulkrax
|
|
200
227
|
def transform_attributes(update: false)
|
201
228
|
@transform_attributes = attributes.slice(*permitted_attributes)
|
202
229
|
@transform_attributes.merge!(file_attributes(update_files)) if with_files
|
203
|
-
@transform_attributes.transform_values! { |v| v == [""] ? [] : v }
|
204
230
|
update ? @transform_attributes.except(:id) : @transform_attributes
|
205
231
|
end
|
206
232
|
|
@@ -81,7 +81,7 @@ module Bulkrax
|
|
81
81
|
# This is adding the reverse relationship, from the child to the parent
|
82
82
|
def collection_parent_work_child
|
83
83
|
child_work_ids = child_records[:works].map(&:id)
|
84
|
-
parent_record.reindex_extent
|
84
|
+
parent_record.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX)
|
85
85
|
|
86
86
|
parent_record.add_member_objects(child_work_ids)
|
87
87
|
ImporterRun.find(importer_run_id).increment!(:processed_relationships, child_work_ids.count) # rubocop:disable Rails/SkipsModelValidations
|
@@ -5,25 +5,27 @@ module Bulkrax
|
|
5
5
|
queue_as :import
|
6
6
|
|
7
7
|
# rubocop:disable Rails/SkipsModelValidations
|
8
|
-
def perform(*
|
9
|
-
entry = Entry.find(
|
8
|
+
def perform(entry_id, run_id, *)
|
9
|
+
entry = Entry.find(entry_id)
|
10
|
+
importer_run = ImporterRun.find(run_id)
|
10
11
|
entry.build
|
11
12
|
if entry.status == "Complete"
|
12
|
-
|
13
|
-
|
14
|
-
ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
|
13
|
+
importer_run.increment!(:processed_records)
|
14
|
+
importer_run.increment!(:processed_works)
|
15
15
|
else
|
16
16
|
# do not retry here because whatever parse error kept you from creating a work will likely
|
17
17
|
# keep preventing you from doing so.
|
18
|
-
|
19
|
-
|
20
|
-
ImporterRun.find(args[1]).decrement!(:enqueued_records) unless ImporterRun.find(args[1]).enqueued_records <= 0 # rubocop:disable Style/IdenticalConditionalBranches
|
18
|
+
importer_run.increment!(:failed_records)
|
19
|
+
importer_run.increment!(:failed_works)
|
21
20
|
end
|
21
|
+
# Regardless of completion or not, we want to decrement the enqueued records.
|
22
|
+
importer_run.decrement!(:enqueued_records) unless importer_run.enqueued_records <= 0
|
23
|
+
|
22
24
|
entry.save!
|
23
|
-
entry.importer.current_run =
|
25
|
+
entry.importer.current_run = importer_run
|
24
26
|
entry.importer.record_status
|
25
27
|
rescue Bulkrax::CollectionsCreatedError
|
26
|
-
reschedule(
|
28
|
+
reschedule(entry_id, run_id)
|
27
29
|
end
|
28
30
|
# rubocop:enable Rails/SkipsModelValidations
|
29
31
|
|
@@ -30,7 +30,7 @@ module Bulkrax
|
|
30
30
|
|
31
31
|
def process_split
|
32
32
|
if self.split.is_a?(TrueClass)
|
33
|
-
@result = @result.split(
|
33
|
+
@result = @result.split(Bulkrax.multi_value_element_split_on)
|
34
34
|
elsif self.split
|
35
35
|
result = @result.split(Regexp.new(self.split))
|
36
36
|
@result = result.map(&:strip)
|
@@ -18,7 +18,7 @@ module Bulkrax
|
|
18
18
|
raise StandardError, 'CSV path empty' if path.blank?
|
19
19
|
CSV.read(path,
|
20
20
|
headers: true,
|
21
|
-
header_converters:
|
21
|
+
header_converters: ->(h) { h.to_sym },
|
22
22
|
encoding: 'utf-8')
|
23
23
|
end
|
24
24
|
|
@@ -81,7 +81,7 @@ module Bulkrax
|
|
81
81
|
def add_file
|
82
82
|
self.parsed_metadata['file'] ||= []
|
83
83
|
if record['file']&.is_a?(String)
|
84
|
-
self.parsed_metadata['file'] = record['file'].split(
|
84
|
+
self.parsed_metadata['file'] = record['file'].split(Bulkrax.multi_value_element_split_on)
|
85
85
|
elsif record['file'].is_a?(Array)
|
86
86
|
self.parsed_metadata['file'] = record['file']
|
87
87
|
end
|
@@ -176,7 +176,7 @@ module Bulkrax
|
|
176
176
|
data = hyrax_record.send(key.to_s)
|
177
177
|
if data.is_a?(ActiveTriples::Relation)
|
178
178
|
if value['join']
|
179
|
-
self.parsed_metadata[key_for_export(key)] = data.map { |d| prepare_export_data(d) }.join(
|
179
|
+
self.parsed_metadata[key_for_export(key)] = data.map { |d| prepare_export_data(d) }.join(Bulkrax.multi_value_element_join_on).to_s
|
180
180
|
else
|
181
181
|
data.each_with_index do |d, i|
|
182
182
|
self.parsed_metadata["#{key_for_export(key)}_#{i + 1}"] = prepare_export_data(d)
|
@@ -236,7 +236,7 @@ module Bulkrax
|
|
236
236
|
|
237
237
|
def handle_join_on_export(key, values, join)
|
238
238
|
if join
|
239
|
-
parsed_metadata[key] = values.join(
|
239
|
+
parsed_metadata[key] = values.join(Bulkrax.multi_value_element_join_on)
|
240
240
|
else
|
241
241
|
values.each_with_index do |value, i|
|
242
242
|
parsed_metadata["#{key}_#{i + 1}"] = value
|
@@ -260,7 +260,7 @@ module Bulkrax
|
|
260
260
|
return [] unless parent_field_mapping.present? && record[parent_field_mapping].present?
|
261
261
|
|
262
262
|
identifiers = []
|
263
|
-
split_references = record[parent_field_mapping].split(
|
263
|
+
split_references = record[parent_field_mapping].split(Bulkrax.multi_value_element_split_on)
|
264
264
|
split_references.each do |c_reference|
|
265
265
|
matching_collection_entries = importerexporter.entries.select do |e|
|
266
266
|
(e.raw_metadata&.[](source_identifier) == c_reference) &&
|
@@ -58,17 +58,26 @@ module Bulkrax
|
|
58
58
|
|
59
59
|
# If field_mapping is empty, setup a default based on the export_properties
|
60
60
|
def mapping
|
61
|
+
# rubocop:disable Style/IfUnlessModifier
|
61
62
|
@mapping ||= if self.field_mapping.blank? || self.field_mapping == [{}]
|
62
63
|
if parser.import_fields.present? || self.field_mapping == [{}]
|
63
|
-
|
64
|
-
parser.import_fields.reject(&:nil?).map do |m|
|
65
|
-
Bulkrax.default_field_mapping.call(m)
|
66
|
-
end.inject(:merge)
|
67
|
-
)
|
64
|
+
default_field_mapping
|
68
65
|
end
|
69
66
|
else
|
70
|
-
self.field_mapping
|
67
|
+
default_field_mapping.merge(self.field_mapping)
|
71
68
|
end
|
69
|
+
|
70
|
+
# rubocop:enable Style/IfUnlessModifier
|
71
|
+
end
|
72
|
+
|
73
|
+
def default_field_mapping
|
74
|
+
return self.field_mapping if parser.import_fields.nil?
|
75
|
+
|
76
|
+
ActiveSupport::HashWithIndifferentAccess.new(
|
77
|
+
parser.import_fields.reject(&:nil?).map do |m|
|
78
|
+
Bulkrax.default_field_mapping.call(m)
|
79
|
+
end.inject(:merge)
|
80
|
+
)
|
72
81
|
end
|
73
82
|
|
74
83
|
def parser_fields
|
@@ -143,17 +152,13 @@ module Bulkrax
|
|
143
152
|
import_objects(['relationship'])
|
144
153
|
end
|
145
154
|
|
155
|
+
DEFAULT_OBJECT_TYPES = %w[collection work file_set relationship].freeze
|
156
|
+
|
146
157
|
def import_objects(types_array = nil)
|
147
158
|
self.only_updates ||= false
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
else
|
152
|
-
types.each do |object_type|
|
153
|
-
self.save if self.new_record? # Object needs to be saved for statuses
|
154
|
-
parser.send("create_#{object_type.pluralize}")
|
155
|
-
end
|
156
|
-
end
|
159
|
+
self.save if self.new_record? # Object needs to be saved for statuses
|
160
|
+
types = types_array || DEFAULT_OBJECT_TYPES
|
161
|
+
parser.create_objects(types)
|
157
162
|
rescue StandardError => e
|
158
163
|
status_info(e)
|
159
164
|
end
|
@@ -59,12 +59,11 @@ module Bulkrax
|
|
59
59
|
def find_collection_ids
|
60
60
|
return self.collection_ids if collections_created?
|
61
61
|
if sets.blank? || parser.collection_name != 'all'
|
62
|
-
# c = Collection.where(Bulkrax.system_identifier_field => importerexporter.unique_collection_identifier(parser.collection_name)).first
|
63
62
|
collection = find_collection(importerexporter.unique_collection_identifier(parser.collection_name))
|
64
63
|
self.collection_ids << collection.id if collection.present? && !self.collection_ids.include?(collection.id)
|
65
64
|
else # All - collections should exist for all sets
|
66
65
|
sets.each do |set|
|
67
|
-
c =
|
66
|
+
c = find_collection(importerexporter.unique_collection_identifier(set.content))
|
68
67
|
self.collection_ids << c.id if c.present? && !self.collection_ids.include?(c.id)
|
69
68
|
end
|
70
69
|
end
|
@@ -6,7 +6,14 @@ module Bulkrax
|
|
6
6
|
::FileSet
|
7
7
|
end
|
8
8
|
|
9
|
+
def file_reference
|
10
|
+
return 'file' if parsed_metadata&.[]('file')&.map(&:present?)&.any?
|
11
|
+
return 'remote_files' if parsed_metadata&.[]('remote_files')&.map(&:present?)&.any?
|
12
|
+
end
|
13
|
+
|
9
14
|
def add_path_to_file
|
15
|
+
return unless file_reference == 'file'
|
16
|
+
|
10
17
|
parsed_metadata['file'].each_with_index do |filename, i|
|
11
18
|
next if filename.blank?
|
12
19
|
|
@@ -22,7 +29,7 @@ module Bulkrax
|
|
22
29
|
end
|
23
30
|
|
24
31
|
def validate_presence_of_filename!
|
25
|
-
return if parsed_metadata&.[](
|
32
|
+
return if parsed_metadata&.[](file_reference)&.map(&:present?)&.any?
|
26
33
|
|
27
34
|
raise StandardError, 'File set must have a filename'
|
28
35
|
end
|
@@ -165,15 +165,16 @@ module Bulkrax
|
|
165
165
|
end
|
166
166
|
|
167
167
|
def factory
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
168
|
+
of = Bulkrax.object_factory || Bulkrax::ObjectFactory
|
169
|
+
@factory ||= of.new(attributes: self.parsed_metadata,
|
170
|
+
source_identifier_value: identifier,
|
171
|
+
work_identifier: parser.work_identifier,
|
172
|
+
related_parents_parsed_mapping: parser.related_parents_parsed_mapping,
|
173
|
+
replace_files: replace_files,
|
174
|
+
user: user,
|
175
|
+
klass: factory_class,
|
176
|
+
importer_run_id: importerexporter.last_run.id,
|
177
|
+
update_files: update_files)
|
177
178
|
end
|
178
179
|
|
179
180
|
def factory_class
|
@@ -2,6 +2,9 @@
|
|
2
2
|
require 'zip'
|
3
3
|
|
4
4
|
module Bulkrax
|
5
|
+
# An abstract class that establishes the API for Bulkrax's import and export parsing.
|
6
|
+
#
|
7
|
+
# @abstract Subclass the Bulkrax::ApplicationParser to create a parser that handles a specific format (e.g. CSV, Bagit, XML, etc).
|
5
8
|
class ApplicationParser # rubocop:disable Metrics/ClassLength
|
6
9
|
attr_accessor :importerexporter, :headers
|
7
10
|
alias importer importerexporter
|
@@ -12,14 +15,21 @@ module Bulkrax
|
|
12
15
|
:exporter_export_path, :exporter_export_zip_path, :importer_unzip_path, :validate_only,
|
13
16
|
to: :importerexporter
|
14
17
|
|
18
|
+
# @todo Convert to `class_attribute :parser_fiels, default: {}`
|
15
19
|
def self.parser_fields
|
16
20
|
{}
|
17
21
|
end
|
18
22
|
|
23
|
+
# @return [TrueClass,FalseClass] this parser does or does not support exports.
|
24
|
+
#
|
25
|
+
# @todo Convert to `class_attribute :export_supported, default: false, instance_predicate: true` and `self << class; alias export_supported? export_supported; end`
|
19
26
|
def self.export_supported?
|
20
27
|
false
|
21
28
|
end
|
22
29
|
|
30
|
+
# @return [TrueClass,FalseClass] this parser does or does not support imports.
|
31
|
+
#
|
32
|
+
# @todo Convert to `class_attribute :import_supported, default: false, instance_predicate: true` and `self << class; alias import_supported? import_supported; end`
|
23
33
|
def self.import_supported?
|
24
34
|
true
|
25
35
|
end
|
@@ -29,49 +39,70 @@ module Bulkrax
|
|
29
39
|
@headers = []
|
30
40
|
end
|
31
41
|
|
32
|
-
# @api
|
42
|
+
# @api public
|
43
|
+
# @abstract Subclass and override {#entry_class} to implement behavior for the parser.
|
33
44
|
def entry_class
|
34
|
-
raise
|
45
|
+
raise NotImplementedError, 'must be defined'
|
35
46
|
end
|
36
47
|
|
37
|
-
# @api
|
48
|
+
# @api public
|
49
|
+
# @abstract Subclass and override {#collection_entry_class} to implement behavior for the parser.
|
38
50
|
def collection_entry_class
|
39
|
-
raise
|
51
|
+
raise NotImplementedError, 'must be defined'
|
40
52
|
end
|
41
53
|
|
42
|
-
# @api
|
54
|
+
# @api public
|
55
|
+
# @abstract Subclass and override {#records} to implement behavior for the parser.
|
43
56
|
def records(_opts = {})
|
44
|
-
raise
|
57
|
+
raise NotImplementedError, 'must be defined'
|
45
58
|
end
|
46
59
|
|
60
|
+
# @return [Symbol] the name of the identifying property in the source system from which we're
|
61
|
+
# importing (e.g. is *not* this application that mounts *this* Bulkrax engine).
|
62
|
+
#
|
63
|
+
# @see #work_identifier
|
64
|
+
# @see https://github.com/samvera-labs/bulkrax/wiki/CSV-Importer#source-identifier Bulkrax Wiki regarding source identifier
|
47
65
|
def source_identifier
|
48
66
|
@source_identifier ||= get_field_mapping_hash_for('source_identifier')&.values&.first&.[]('from')&.first&.to_sym || :source_identifier
|
49
67
|
end
|
50
68
|
|
69
|
+
# @return [Symbol] the name of the identifying property for the system which we're importing
|
70
|
+
# into (e.g. the application that mounts *this* Bulkrax engine)
|
71
|
+
# @see #source_identifier
|
51
72
|
def work_identifier
|
52
73
|
@work_identifier ||= get_field_mapping_hash_for('source_identifier')&.keys&.first&.to_sym || :source
|
53
74
|
end
|
54
75
|
|
76
|
+
# @return [String]
|
55
77
|
def generated_metadata_mapping
|
56
78
|
@generated_metadata_mapping ||= 'generated'
|
57
79
|
end
|
58
80
|
|
81
|
+
# @return [String, NilClass]
|
82
|
+
# @see #related_parents_raw_mapping
|
59
83
|
def related_parents_raw_mapping
|
60
84
|
@related_parents_raw_mapping ||= get_field_mapping_hash_for('related_parents_field_mapping')&.values&.first&.[]('from')&.first
|
61
85
|
end
|
62
86
|
|
87
|
+
# @return [String]
|
88
|
+
# @see #related_parents_field_mapping
|
63
89
|
def related_parents_parsed_mapping
|
64
90
|
@related_parents_parsed_mapping ||= (get_field_mapping_hash_for('related_parents_field_mapping')&.keys&.first || 'parents')
|
65
91
|
end
|
66
92
|
|
93
|
+
# @return [String, NilClass]
|
94
|
+
# @see #related_children_parsed_mapping
|
67
95
|
def related_children_raw_mapping
|
68
96
|
@related_children_raw_mapping ||= get_field_mapping_hash_for('related_children_field_mapping')&.values&.first&.[]('from')&.first
|
69
97
|
end
|
70
98
|
|
99
|
+
# @return [String]
|
100
|
+
# @see #related_children_raw_mapping
|
71
101
|
def related_children_parsed_mapping
|
72
102
|
@related_children_parsed_mapping ||= (get_field_mapping_hash_for('related_children_field_mapping')&.keys&.first || 'children')
|
73
103
|
end
|
74
104
|
|
105
|
+
# @api private
|
75
106
|
def get_field_mapping_hash_for(key)
|
76
107
|
return instance_variable_get("@#{key}_hash") if instance_variable_get("@#{key}_hash").present?
|
77
108
|
|
@@ -85,6 +116,7 @@ module Bulkrax
|
|
85
116
|
instance_variable_get("@#{key}_hash")
|
86
117
|
end
|
87
118
|
|
119
|
+
# @return [Array<String>]
|
88
120
|
def model_field_mappings
|
89
121
|
model_mappings = Bulkrax.field_mappings[self.class.to_s]&.dig('model', :from) || []
|
90
122
|
model_mappings |= ['model']
|
@@ -92,6 +124,7 @@ module Bulkrax
|
|
92
124
|
model_mappings
|
93
125
|
end
|
94
126
|
|
127
|
+
# @return [String]
|
95
128
|
def perform_method
|
96
129
|
if self.validate_only
|
97
130
|
'perform_now'
|
@@ -100,29 +133,55 @@ module Bulkrax
|
|
100
133
|
end
|
101
134
|
end
|
102
135
|
|
136
|
+
# The visibility of the record. Acceptable values are: "open", "embaro", "lease", "authenticated", "restricted". The default is "open"
|
137
|
+
#
|
138
|
+
# @return [String]
|
139
|
+
# @see https://github.com/samvera/hydra-head/blob/main/hydra-access-controls/app/models/concerns/hydra/access_controls/access_right.rb Hydra::AccessControls::AccessRight for details on the range of values.
|
140
|
+
# @see https://github.com/samvera/hyrax/blob/bd2bcffc33e183904be2c175367648815f25bc2b/app/services/hyrax/visibility_intention.rb Hyrax::VisibilityIntention for how we process the visibility.
|
103
141
|
def visibility
|
104
142
|
@visibility ||= self.parser_fields['visibility'] || 'open'
|
105
143
|
end
|
106
144
|
|
145
|
+
# @api public
|
146
|
+
#
|
147
|
+
# @param types [Array<Symbol>] the types of objects that we'll create.
|
148
|
+
#
|
149
|
+
# @see Bulkrax::Importer::DEFAULT_OBJECT_TYPES
|
150
|
+
# @see #create_collections
|
151
|
+
# @see #create_works
|
152
|
+
# @see #create_file_sets
|
153
|
+
# @see #create_relationships
|
154
|
+
def create_objects(types = [])
|
155
|
+
types.each do |object_type|
|
156
|
+
parser.send("create_#{object_type.pluralize}")
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
# @abstract Subclass and override {#create_collections} to implement behavior for the parser.
|
107
161
|
def create_collections
|
108
|
-
raise
|
162
|
+
raise NotImplementedError, 'must be defined' if importer?
|
109
163
|
end
|
110
164
|
|
165
|
+
# @abstract Subclass and override {#create_works} to implement behavior for the parser.
|
111
166
|
def create_works
|
112
|
-
raise
|
167
|
+
raise NotImplementedError, 'must be defined' if importer?
|
113
168
|
end
|
114
169
|
|
170
|
+
# @abstract Subclass and override {#create_file_sets} to implement behavior for the parser.
|
115
171
|
def create_file_sets
|
116
|
-
raise
|
172
|
+
raise NotImplementedError, 'must be defined' if importer?
|
117
173
|
end
|
118
174
|
|
175
|
+
# @abstract Subclass and override {#create_relationships} to implement behavior for the parser.
|
119
176
|
def create_relationships
|
120
|
-
raise
|
177
|
+
raise NotImplementedError, 'must be defined' if importer?
|
121
178
|
end
|
122
179
|
|
123
180
|
# Optional, define if using browse everything for file upload
|
124
181
|
def retrieve_cloud_files(files); end
|
125
182
|
|
183
|
+
# @param file [#path, #original_filename] the file object that with the relevant data for the
|
184
|
+
# import.
|
126
185
|
def write_import_file(file)
|
127
186
|
path = File.join(path_for_import, file.original_filename)
|
128
187
|
FileUtils.mv(
|
@@ -133,6 +192,8 @@ module Bulkrax
|
|
133
192
|
end
|
134
193
|
|
135
194
|
# Base path for imported and exported files
|
195
|
+
# @param [String]
|
196
|
+
# @return [String] the base path for files that this parser will "parse"
|
136
197
|
def base_path(type = 'import')
|
137
198
|
# account for multiple versions of hyku
|
138
199
|
is_multitenant = ENV['HYKU_MULTITENANT'] == 'true' || ENV['SETTINGS__MULTITENANCY__ENABLED'] == 'true'
|
@@ -141,41 +202,48 @@ module Bulkrax
|
|
141
202
|
|
142
203
|
# Path where we'll store the import metadata and files
|
143
204
|
# this is used for uploaded and cloud files
|
205
|
+
# @return [String]
|
144
206
|
def path_for_import
|
145
207
|
@path_for_import = File.join(base_path, importerexporter.path_string)
|
146
208
|
FileUtils.mkdir_p(@path_for_import) unless File.exist?(@path_for_import)
|
147
209
|
@path_for_import
|
148
210
|
end
|
149
211
|
|
212
|
+
# @abstract Subclass and override {#setup_export_file} to implement behavior for the parser.
|
150
213
|
def setup_export_file
|
151
|
-
raise
|
214
|
+
raise NotImplementedError, 'must be defined' if exporter?
|
152
215
|
end
|
153
216
|
|
217
|
+
# @abstract Subclass and override {#write_files} to implement behavior for the parser.
|
154
218
|
def write_files
|
155
|
-
raise
|
219
|
+
raise NotImplementedError, 'must be defined' if exporter?
|
156
220
|
end
|
157
221
|
|
222
|
+
# @return [TrueClass,FalseClass]
|
158
223
|
def importer?
|
159
224
|
importerexporter.is_a?(Bulkrax::Importer)
|
160
225
|
end
|
161
226
|
|
227
|
+
# @return [TrueClass,FalseClass]
|
162
228
|
def exporter?
|
163
229
|
importerexporter.is_a?(Bulkrax::Exporter)
|
164
230
|
end
|
165
231
|
|
166
232
|
# @param limit [Integer] limit set on the importerexporter
|
167
233
|
# @param index [Integer] index of current iteration
|
168
|
-
# @return [
|
234
|
+
# @return [TrueClass,FalseClass]
|
169
235
|
def limit_reached?(limit, index)
|
170
236
|
return false if limit.nil? || limit.zero? # no limit
|
171
237
|
index >= limit
|
172
238
|
end
|
173
239
|
|
174
240
|
# Override to add specific validations
|
241
|
+
# @return [TrueClass,FalseClass]
|
175
242
|
def valid_import?
|
176
243
|
true
|
177
244
|
end
|
178
245
|
|
246
|
+
# @return [TrueClass,FalseClass]
|
179
247
|
def record_has_source_identifier(record, index)
|
180
248
|
if record[source_identifier].blank?
|
181
249
|
if Bulkrax.fill_in_blank_source_identifiers.present?
|
@@ -199,6 +267,7 @@ module Bulkrax
|
|
199
267
|
end
|
200
268
|
# rubocop:enable Rails/SkipsModelValidations
|
201
269
|
|
270
|
+
# @return [Array<String>]
|
202
271
|
def required_elements
|
203
272
|
if Bulkrax.fill_in_blank_source_identifiers
|
204
273
|
['title']
|
@@ -287,12 +356,14 @@ module Bulkrax
|
|
287
356
|
end
|
288
357
|
|
289
358
|
# Path for the import
|
359
|
+
# @return [String]
|
290
360
|
def import_file_path
|
291
361
|
@import_file_path ||= real_import_file_path
|
292
362
|
end
|
293
363
|
|
294
364
|
private
|
295
365
|
|
366
|
+
# @return [String]
|
296
367
|
def real_import_file_path
|
297
368
|
return importer_unzip_path if file? && zip?
|
298
369
|
parser_fields['import_file_path']
|
@@ -180,7 +180,7 @@ module Bulkrax
|
|
180
180
|
end
|
181
181
|
|
182
182
|
def current_work_ids
|
183
|
-
ActiveSupport::
|
183
|
+
ActiveSupport::Deprecation.warn('Bulkrax::CsvParser#current_work_ids will be replaced with #current_record_ids in version 3.0')
|
184
184
|
current_record_ids
|
185
185
|
end
|
186
186
|
|
@@ -440,7 +440,7 @@ module Bulkrax
|
|
440
440
|
file_mapping = Bulkrax.field_mappings.dig(self.class.to_s, 'file', :from)&.first&.to_sym || :file
|
441
441
|
next if r[file_mapping].blank?
|
442
442
|
|
443
|
-
r[file_mapping].split(
|
443
|
+
r[file_mapping].split(Bulkrax.multi_value_element_split_on).map do |f|
|
444
444
|
file = File.join(path_to_files, f.tr(' ', '_'))
|
445
445
|
if File.exist?(file) # rubocop:disable Style/GuardClause
|
446
446
|
file
|
@@ -468,7 +468,7 @@ module Bulkrax
|
|
468
468
|
entry_uid ||= if Bulkrax.fill_in_blank_source_identifiers.present?
|
469
469
|
Bulkrax.fill_in_blank_source_identifiers.call(self, records.find_index(collection_hash))
|
470
470
|
else
|
471
|
-
collection_hash[:title].split(
|
471
|
+
collection_hash[:title].split(Bulkrax.multi_value_element_split_on).first
|
472
472
|
end
|
473
473
|
|
474
474
|
entry_uid
|
@@ -13,8 +13,7 @@ module Bulkrax
|
|
13
13
|
def client
|
14
14
|
@client ||= OAI::Client.new(importerexporter.parser_fields['base_url'],
|
15
15
|
headers: headers,
|
16
|
-
parser: 'libxml'
|
17
|
-
metadata_prefix: importerexporter.parser_fields['metadata_prefix'])
|
16
|
+
parser: 'libxml')
|
18
17
|
rescue StandardError
|
19
18
|
raise OAIError
|
20
19
|
end
|
@@ -32,6 +31,7 @@ module Bulkrax
|
|
32
31
|
end
|
33
32
|
|
34
33
|
def records(opts = {})
|
34
|
+
opts[:metadata_prefix] ||= importerexporter.parser_fields['metadata_prefix']
|
35
35
|
opts[:set] = collection_name unless collection_name == 'all'
|
36
36
|
|
37
37
|
opts[:from] = importerexporter&.last_imported_at&.strftime("%Y-%m-%d") if importerexporter.last_imported_at && only_updates
|
@@ -0,0 +1,107 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Bulkrax
|
3
|
+
# This module is rather destructive; it will break relationships between the works, file sets, and
|
4
|
+
# collections that were imported via an importer. You probably don't want to run this on your
|
5
|
+
# data, except in the case where you have been testing a Bulkrax::Importer, the parsers and
|
6
|
+
# mappings. Then, you might have relationships that you want to remove.
|
7
|
+
#
|
8
|
+
# tl;dr - Caution this will break things!
|
9
|
+
class RemoveRelationshipsForImporter
|
10
|
+
# @api public
|
11
|
+
#
|
12
|
+
# Remove the relationships of the works and collections for all of the Bulkrax::Entry records
|
13
|
+
# associated with the given Bulkrax::Importer.
|
14
|
+
#
|
15
|
+
# @param importer [Bulkrax::Importer]
|
16
|
+
# @param with_progress_bar [Boolean]
|
17
|
+
def self.break_relationships_for!(importer:, with_progress_bar: false)
|
18
|
+
entries = importer.entries.select(&:succeeded?)
|
19
|
+
progress_bar = build_progress_bar_for(with_progress_bar: with_progress_bar, entries: entries)
|
20
|
+
new(progress_bar: progress_bar, entries: entries).break_relationships!
|
21
|
+
end
|
22
|
+
|
23
|
+
# @api private
|
24
|
+
#
|
25
|
+
# A null object that conforms to this class's use of a progress bar.
|
26
|
+
module NullProgressBar
|
27
|
+
def self.increment; end
|
28
|
+
end
|
29
|
+
|
30
|
+
# @api private
|
31
|
+
#
|
32
|
+
# @return [#increment]
|
33
|
+
def self.build_progress_bar_for(with_progress_bar:, entries:)
|
34
|
+
return NullProgressBar unless with_progress_bar
|
35
|
+
|
36
|
+
begin
|
37
|
+
require 'ruby-progressbar'
|
38
|
+
ProgessBar.create(total: entries.count)
|
39
|
+
rescue LoadError
|
40
|
+
Rails.logger.info("Using NullProgressBar because ProgressBar is not available due to a LoadError.")
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
# @param entries [#each]
|
45
|
+
# @param progress_bar [#increment]
|
46
|
+
def initialize(entries:, progress_bar:)
|
47
|
+
@progress_bar = progress_bar
|
48
|
+
@entries = entries
|
49
|
+
end
|
50
|
+
|
51
|
+
attr_reader :entries, :progress_bar
|
52
|
+
|
53
|
+
def break_relationships!
|
54
|
+
entries.each do |entry|
|
55
|
+
progress_bar.increment
|
56
|
+
|
57
|
+
obj = entry.factory.find
|
58
|
+
next if obj.is_a?(FileSet) # FileSets must be attached to a Work
|
59
|
+
|
60
|
+
if obj.is_a?(Collection)
|
61
|
+
remove_relationships_from_collection(obj)
|
62
|
+
else
|
63
|
+
remove_relationships_from_work(obj)
|
64
|
+
end
|
65
|
+
|
66
|
+
obj.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX)
|
67
|
+
obj.save!
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def remove_relationships_from_collection(collection)
|
72
|
+
# Remove child work relationships
|
73
|
+
collection.member_works.each do |work|
|
74
|
+
change = work.member_of_collections.delete(collection)
|
75
|
+
work.save! if change.present?
|
76
|
+
end
|
77
|
+
|
78
|
+
# Remove parent collection relationships
|
79
|
+
collection.member_of_collections.each do |parent_col|
|
80
|
+
Hyrax::Collections::NestedCollectionPersistenceService
|
81
|
+
.remove_nested_relationship_for(parent: parent_col, child: collection)
|
82
|
+
end
|
83
|
+
|
84
|
+
# Remove child collection relationships
|
85
|
+
collection.member_collections.each do |child_col|
|
86
|
+
Hyrax::Collections::NestedCollectionPersistenceService
|
87
|
+
.remove_nested_relationship_for(parent: collection, child: child_col)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def remove_relationships_from_work(work)
|
92
|
+
# Remove parent collection relationships
|
93
|
+
work.member_of_collections = []
|
94
|
+
|
95
|
+
# Remove parent work relationships
|
96
|
+
work.member_of_works.each do |parent_work|
|
97
|
+
parent_work.members.delete(work)
|
98
|
+
parent_work.save!
|
99
|
+
end
|
100
|
+
|
101
|
+
# Remove child work relationships
|
102
|
+
work.member_works.each do |child_work|
|
103
|
+
work.member_works.delete(child_work)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
@@ -29,7 +29,7 @@
|
|
29
29
|
|
30
30
|
<%= form.input :export_source_importer,
|
31
31
|
label: t('bulkrax.exporter.labels.importer'),
|
32
|
-
|
32
|
+
required: true,
|
33
33
|
prompt: 'Select from the list',
|
34
34
|
label_html: { class: 'importer export-source-option hidden' },
|
35
35
|
input_html: { class: 'importer export-source-option hidden' },
|
@@ -38,7 +38,7 @@
|
|
38
38
|
<%= form.input :export_source_collection,
|
39
39
|
prompt: 'Start typing ...',
|
40
40
|
label: t('bulkrax.exporter.labels.collection'),
|
41
|
-
|
41
|
+
required: true,
|
42
42
|
placeholder: @collection&.title&.first,
|
43
43
|
label_html: { class: 'collection export-source-option hidden' },
|
44
44
|
input_html: {
|
@@ -52,7 +52,7 @@
|
|
52
52
|
|
53
53
|
<%= form.input :export_source_worktype,
|
54
54
|
label: t('bulkrax.exporter.labels.worktype'),
|
55
|
-
|
55
|
+
required: true,
|
56
56
|
prompt: 'Select from the list',
|
57
57
|
label_html: { class: 'worktype export-source-option hidden' },
|
58
58
|
input_html: { class: 'worktype export-source-option hidden' },
|
@@ -14,7 +14,7 @@
|
|
14
14
|
<% entries.each do |e| %>
|
15
15
|
<% entry_path = item.class.to_s.include?('Importer') ? bulkrax.importer_entry_path(item.id, e.id) : bulkrax.exporter_entry_path(item.id, e.id) %>
|
16
16
|
<tr>
|
17
|
-
<td><%= link_to e.identifier,
|
17
|
+
<td><%= link_to e.identifier, entry_path %></td>
|
18
18
|
<td><%= e.id %></td>
|
19
19
|
<% if e.status == "Complete" %>
|
20
20
|
<td><span class="glyphicon glyphicon-ok" style="color: green;"></span> <%= e.status %></td>
|
@@ -24,12 +24,12 @@
|
|
24
24
|
<td><span class="glyphicon glyphicon-remove" style="color: <%= e.status == 'Deleted' ? 'green' : 'red' %>;"></span> <%= e.status %></td>
|
25
25
|
<% end %>
|
26
26
|
<% if e.last_error.present? %>
|
27
|
-
<td><%= link_to e.last_error.dig("error_class"),
|
27
|
+
<td><%= link_to e.last_error.dig("error_class"), entry_path %></td>
|
28
28
|
<% else %>
|
29
29
|
<td></td>
|
30
30
|
<% end %>
|
31
31
|
<td><%= e.status_at %></td>
|
32
|
-
<td><%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'),
|
32
|
+
<td><%= link_to raw('<span class="glyphicon glyphicon-info-sign"></span>'), entry_path %></td>
|
33
33
|
</tr>
|
34
34
|
<% end %>
|
35
35
|
</tbody>
|
@@ -1,6 +1,8 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
<% if ENV.fetch('HYKU_BULKRAX_ENABLED', 'true') == 'true' %>
|
2
|
+
<%= menu.nav_link(bulkrax.importers_path) do %>
|
3
|
+
<span class="fa fa-cloud-upload" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('bulkrax.admin.sidebar.importers') %></span>
|
4
|
+
<% end %>
|
5
|
+
<%= menu.nav_link(bulkrax.exporters_path) do %>
|
6
|
+
<span class="fa fa-cloud-download" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('bulkrax.admin.sidebar.exporters') %></span>
|
7
|
+
<% end %>
|
3
8
|
<% end %>
|
4
|
-
<%= menu.nav_link(bulkrax.exporters_path) do %>
|
5
|
-
<span class="fa fa-cloud-download" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('bulkrax.admin.sidebar.exporters') %></span>
|
6
|
-
<% end %>
|
@@ -1,19 +1,27 @@
|
|
1
|
-
<li class="h5"><%= t('hyrax.admin.sidebar.repository_objects') %></li>
|
1
|
+
<li class="h5 nav-item"><%= t('hyrax.admin.sidebar.repository_objects') %></li>
|
2
2
|
|
3
|
-
<%= menu.nav_link(hyrax.my_collections_path,
|
4
|
-
|
5
|
-
|
6
|
-
|
3
|
+
<%= menu.nav_link(hyrax.my_collections_path,
|
4
|
+
class: "nav-link",
|
5
|
+
onclick: "dontChangeAccordion(event);",
|
6
|
+
also_active_for: hyrax.dashboard_collections_path) do %>
|
7
|
+
<span class="fa fa-folder-open" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('hyrax.admin.sidebar.collections') %></span>
|
8
|
+
<% end %>
|
7
9
|
|
8
|
-
<%= menu.nav_link(hyrax.my_works_path,
|
9
|
-
|
10
|
-
|
11
|
-
|
10
|
+
<%= menu.nav_link(hyrax.my_works_path,
|
11
|
+
class: "nav-link",
|
12
|
+
onclick: "dontChangeAccordion(event);",
|
13
|
+
also_active_for: hyrax.dashboard_works_path) do %>
|
14
|
+
<span class="fa fa-file" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('hyrax.admin.sidebar.works') %></span>
|
15
|
+
<% end %>
|
12
16
|
|
13
|
-
|
14
|
-
|
15
|
-
<%
|
17
|
+
<% if ::Hyrax::DashboardController&.respond_to?(:sidebar_partials) %>
|
18
|
+
<%= render 'hyrax/dashboard/sidebar/menu_partials', menu: menu, section: :repository_content %>
|
19
|
+
<% else %>
|
20
|
+
<%= menu.nav_link(bulkrax.importers_path) do %>
|
21
|
+
<span class="fa fa-cloud-upload" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('bulkrax.admin.sidebar.importers') %></span>
|
22
|
+
<% end %>
|
16
23
|
|
17
|
-
<%= menu.nav_link(bulkrax.exporters_path) do %>
|
18
|
-
|
19
|
-
<% end %>
|
24
|
+
<%= menu.nav_link(bulkrax.exporters_path) do %>
|
25
|
+
<span class="fa fa-cloud-download" aria-hidden="true"></span> <span class="sidebar-action-text"><%= t('bulkrax.admin.sidebar.exporters') %></span>
|
26
|
+
<% end %>
|
27
|
+
<% end %>
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class RenameChildrenCountersToRelationships < ActiveRecord::Migration[5.
|
1
|
+
class RenameChildrenCountersToRelationships < ActiveRecord::Migration[5.1]
|
2
2
|
def change
|
3
3
|
rename_column :bulkrax_importer_runs, :processed_children, :processed_relationships unless column_exists?(:bulkrax_importer_runs, :processed_relationships)
|
4
4
|
rename_column :bulkrax_importer_runs, :failed_children, :failed_relationships unless column_exists?(:bulkrax_importer_runs, :failed_relationships)
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class AddFileSetCountersToImporterRuns < ActiveRecord::Migration[5.
|
1
|
+
class AddFileSetCountersToImporterRuns < ActiveRecord::Migration[5.1]
|
2
2
|
def change
|
3
3
|
add_column :bulkrax_importer_runs, :processed_file_sets, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :processed_file_sets)
|
4
4
|
add_column :bulkrax_importer_runs, :failed_file_sets, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :failed_file_sets)
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class AddImportAttemptsToEntries < ActiveRecord::Migration[5.
|
1
|
+
class AddImportAttemptsToEntries < ActiveRecord::Migration[5.1]
|
2
2
|
def change
|
3
3
|
add_column :bulkrax_entries, :import_attempts, :integer, default: 0 unless column_exists?(:bulkrax_entries, :import_attempts)
|
4
4
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class AddWorkCountersToImporterRuns < ActiveRecord::Migration[5.
|
1
|
+
class AddWorkCountersToImporterRuns < ActiveRecord::Migration[5.1]
|
2
2
|
def change
|
3
3
|
add_column :bulkrax_importer_runs, :processed_works, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :processed_works)
|
4
4
|
add_column :bulkrax_importer_runs, :failed_works, :integer, default: 0 unless column_exists?(:bulkrax_importer_runs, :failed_works)
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class AddOrderToBulkraxPendingRelationships < ActiveRecord::Migration[5.
|
1
|
+
class AddOrderToBulkraxPendingRelationships < ActiveRecord::Migration[5.1]
|
2
2
|
def change
|
3
3
|
add_column :bulkrax_pending_relationships, :order, :integer, default: 0 unless column_exists?(:bulkrax_pending_relationships, :order)
|
4
4
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class AddIncludeThumbnailsToBulkraxExporters < ActiveRecord::Migration[5.
|
1
|
+
class AddIncludeThumbnailsToBulkraxExporters < ActiveRecord::Migration[5.1]
|
2
2
|
def change
|
3
3
|
add_column :bulkrax_exporters, :include_thumbnails, :boolean, default: false unless column_exists?(:bulkrax_exporters, :include_thumbnails)
|
4
4
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class AddGeneratedMetadataToBulkraxExporters < ActiveRecord::Migration[5.
|
1
|
+
class AddGeneratedMetadataToBulkraxExporters < ActiveRecord::Migration[5.1]
|
2
2
|
def change
|
3
3
|
add_column :bulkrax_exporters, :generated_metadata, :boolean, default: false unless column_exists?(:bulkrax_exporters, :generated_metadata)
|
4
4
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
class RenameBulkraxImporterRunToImporterRun < ActiveRecord::Migration[5.
|
1
|
+
class RenameBulkraxImporterRunToImporterRun < ActiveRecord::Migration[5.1]
|
2
2
|
def up
|
3
3
|
if column_exists?(:bulkrax_pending_relationships, :bulkrax_importer_run_id)
|
4
4
|
remove_foreign_key :bulkrax_pending_relationships, :bulkrax_importer_runs
|
data/lib/bulkrax/version.rb
CHANGED
data/lib/bulkrax.rb
CHANGED
@@ -1,25 +1,29 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "bulkrax/version"
|
3
4
|
require "bulkrax/engine"
|
4
5
|
require 'active_support/all'
|
5
6
|
|
6
7
|
module Bulkrax
|
7
8
|
class << self
|
8
|
-
mattr_accessor :
|
9
|
-
:default_work_type,
|
9
|
+
mattr_accessor :api_definition,
|
10
10
|
:default_field_mapping,
|
11
|
+
:default_work_type,
|
12
|
+
:export_path,
|
13
|
+
:field_mappings,
|
11
14
|
:fill_in_blank_source_identifiers,
|
12
15
|
:generated_metadata_mapping,
|
16
|
+
:import_path,
|
17
|
+
:multi_value_element_join_on,
|
18
|
+
:multi_value_element_split_on,
|
19
|
+
:object_factory,
|
20
|
+
:parsers,
|
21
|
+
:qa_controlled_properties,
|
13
22
|
:related_children_field_mapping,
|
14
23
|
:related_parents_field_mapping,
|
15
|
-
:reserved_properties,
|
16
|
-
:qa_controlled_properties,
|
17
|
-
:field_mappings,
|
18
|
-
:import_path,
|
19
|
-
:export_path,
|
20
24
|
:removed_image_path,
|
21
|
-
:
|
22
|
-
:
|
25
|
+
:reserved_properties,
|
26
|
+
:server_name
|
23
27
|
|
24
28
|
self.parsers = [
|
25
29
|
{ name: "OAI - Dublin Core", class_name: "Bulkrax::OaiDcParser", partial: "oai_fields" },
|
@@ -29,8 +33,8 @@ module Bulkrax
|
|
29
33
|
{ name: "XML", class_name: "Bulkrax::XmlParser", partial: "xml_fields" }
|
30
34
|
]
|
31
35
|
|
32
|
-
self.import_path = 'tmp/imports'
|
33
|
-
self.export_path = 'tmp/exports'
|
36
|
+
self.import_path = Bulkrax.import_path || 'tmp/imports'
|
37
|
+
self.export_path = Bulkrax.export_path || 'tmp/exports'
|
34
38
|
self.removed_image_path = Bulkrax::Engine.root.join('spec', 'fixtures', 'removed.png').to_s
|
35
39
|
self.server_name = 'bulkrax@example.com'
|
36
40
|
|
@@ -137,6 +141,29 @@ module Bulkrax
|
|
137
141
|
)
|
138
142
|
end
|
139
143
|
|
144
|
+
DEFAULT_MULTI_VALUE_ELEMENT_JOIN_ON = ' | '
|
145
|
+
# Specify the delimiter for joining an attribute's multi-value array into a string.
|
146
|
+
#
|
147
|
+
# @note the specific delimeter should likely be present in the multi_value_element_split_on
|
148
|
+
# expression.
|
149
|
+
def self.multi_value_element_join_on
|
150
|
+
@multi_value_element_join_on ||= DEFAULT_MULTI_VALUE_ELEMENT_JOIN_ON
|
151
|
+
end
|
152
|
+
|
153
|
+
DEFAULT_MULTI_VALUE_ELEMENT_SPLIT_ON = /\s*[:;|]\s*/.freeze
|
154
|
+
# @return [RegexClass] the regular express to use to "split" an attribute's values. If set to
|
155
|
+
# `true` use the DEFAULT_MULTI_VALUE_ELEMENT_JOIN_ON.
|
156
|
+
#
|
157
|
+
# @note The "true" value is to preserve backwards compatibility.
|
158
|
+
# @see DEFAULT_MULTI_VALUE_ELEMENT_JOIN_ON
|
159
|
+
def self.multi_value_element_split_on
|
160
|
+
if @multi_value_element_join_on.is_a?(TrueClass)
|
161
|
+
DEFAULT_MULTI_VALUE_ELEMENT_SPLIT_ON
|
162
|
+
else
|
163
|
+
@multi_value_element_split_on ||= DEFAULT_MULTI_VALUE_ELEMENT_SPLIT_ON
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
140
167
|
# this function maps the vars from your app into your engine
|
141
168
|
def self.setup
|
142
169
|
yield self
|
@@ -10,6 +10,9 @@ Bulkrax.setup do |config|
|
|
10
10
|
# Default is the first returned by Hyrax.config.curation_concerns
|
11
11
|
# config.default_work_type = MyWork
|
12
12
|
|
13
|
+
# Factory Class to use when generating and saving objects
|
14
|
+
config.object_factory = Bulkrax::ObjectFactory
|
15
|
+
|
13
16
|
# Path to store pending imports
|
14
17
|
# config.import_path = 'tmp/imports'
|
15
18
|
|
@@ -67,6 +70,13 @@ Bulkrax.setup do |config|
|
|
67
70
|
# is controlled by the active terms in config/authorities/rights_statements.yml
|
68
71
|
# Defaults: 'rights_statement' and 'license'
|
69
72
|
# config.qa_controlled_properties += ['my_field']
|
73
|
+
|
74
|
+
# Specify the delimiter regular expression for splitting an attribute's values into a multi-value array.
|
75
|
+
# config.multi_value_element_split_on = //\s*[:;|]\s*/.freeze
|
76
|
+
|
77
|
+
# Specify the delimiter for joining an attribute's multi-value array into a string. Note: the
|
78
|
+
# specific delimeter should likely be present in the multi_value_element_split_on expression.
|
79
|
+
# config.multi_value_element_join_on = ' | '
|
70
80
|
end
|
71
81
|
|
72
82
|
# Sidebar for hyrax 3+ support
|
@@ -0,0 +1,65 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
namespace :hyrax do
|
4
|
+
namespace :reset do
|
5
|
+
desc "Delete Bulkrax importers in addition to the reset below"
|
6
|
+
task importers_et_all: :works_and_collections do
|
7
|
+
# sometimes re-running an existing importer causes issues
|
8
|
+
# and you may need to create new ones or delete the existing ones
|
9
|
+
Bulkrax::Importer.delete_all
|
10
|
+
end
|
11
|
+
|
12
|
+
desc 'Reset fedora / solr and corresponding database tables w/o clearing other active record tables like users'
|
13
|
+
task works_and_collections: [:environment] do
|
14
|
+
confirm('You are about to delete all works and collections, this is not reversable!')
|
15
|
+
require 'active_fedora/cleaner'
|
16
|
+
ActiveFedora::Cleaner.clean!
|
17
|
+
Hyrax::PermissionTemplateAccess.delete_all
|
18
|
+
Hyrax::PermissionTemplate.delete_all
|
19
|
+
Bulkrax::PendingRelationship.delete_all
|
20
|
+
Bulkrax::Entry.delete_all
|
21
|
+
Bulkrax::ImporterRun.delete_all
|
22
|
+
Bulkrax::Status.delete_all
|
23
|
+
# Remove sipity methods, everything but sipity roles
|
24
|
+
Sipity::Workflow.delete_all
|
25
|
+
Sipity::EntitySpecificResponsibility.delete_all
|
26
|
+
Sipity::Comment.delete_all
|
27
|
+
Sipity::Entity.delete_all
|
28
|
+
Sipity::WorkflowRole.delete_all
|
29
|
+
Sipity::WorkflowResponsibility.delete_all
|
30
|
+
Sipity::Agent.delete_all
|
31
|
+
Mailboxer::Receipt.destroy_all
|
32
|
+
Mailboxer::Notification.delete_all
|
33
|
+
Mailboxer::Conversation::OptOut.delete_all
|
34
|
+
Mailboxer::Conversation.delete_all
|
35
|
+
AccountElevator.switch!('single.tenant.default')
|
36
|
+
# we need to wait till Fedora is done with its cleanup
|
37
|
+
# otherwise creating the admin set will fail
|
38
|
+
while AdminSet.exists?(AdminSet::DEFAULT_ID)
|
39
|
+
puts 'waiting for delete to finish before reinitializing Fedora'
|
40
|
+
sleep 20
|
41
|
+
end
|
42
|
+
|
43
|
+
Hyrax::CollectionType.find_or_create_default_collection_type
|
44
|
+
Hyrax::CollectionType.find_or_create_admin_set_type
|
45
|
+
AdminSet.find_or_create_default_admin_set_id
|
46
|
+
|
47
|
+
collection_types = Hyrax::CollectionType.all
|
48
|
+
collection_types.each do |c|
|
49
|
+
next unless c.title.match?(/^translation missing/)
|
50
|
+
oldtitle = c.title
|
51
|
+
c.title = I18n.t(c.title.gsub("translation missing: en.", ''))
|
52
|
+
c.save
|
53
|
+
puts "#{oldtitle} changed to #{c.title}"
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def confirm(action)
|
58
|
+
return if ENV['RESET_CONFIRMED'].present?
|
59
|
+
confirm_token = rand(36**6).to_s(36)
|
60
|
+
STDOUT.puts "#{action} Enter '#{confirm_token}' to confirm:"
|
61
|
+
input = STDIN.gets.chomp
|
62
|
+
raise "Aborting. You entered #{input}" unless input == confirm_token
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bulkrax
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rob Kaufman
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-12-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|
@@ -328,6 +328,7 @@ files:
|
|
328
328
|
- app/parsers/bulkrax/oai_dc_parser.rb
|
329
329
|
- app/parsers/bulkrax/oai_qualified_dc_parser.rb
|
330
330
|
- app/parsers/bulkrax/xml_parser.rb
|
331
|
+
- app/services/bulkrax/remove_relationships_for_importer.rb
|
331
332
|
- app/views/bulkrax/entries/_parsed_metadata.html.erb
|
332
333
|
- app/views/bulkrax/entries/_raw_metadata.html.erb
|
333
334
|
- app/views/bulkrax/entries/show.html.erb
|
@@ -404,6 +405,7 @@ files:
|
|
404
405
|
- lib/generators/bulkrax/templates/config/bulkrax_api.yml
|
405
406
|
- lib/generators/bulkrax/templates/config/initializers/bulkrax.rb
|
406
407
|
- lib/tasks/bulkrax_tasks.rake
|
408
|
+
- lib/tasks/reset.rake
|
407
409
|
homepage: https://github.com/samvera-labs/bulkrax
|
408
410
|
licenses:
|
409
411
|
- Apache-2.0
|
@@ -423,7 +425,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
423
425
|
- !ruby/object:Gem::Version
|
424
426
|
version: '0'
|
425
427
|
requirements: []
|
426
|
-
rubygems_version: 3.
|
428
|
+
rubygems_version: 3.0.3
|
427
429
|
signing_key:
|
428
430
|
specification_version: 4
|
429
431
|
summary: Import and export tool for Hyrax and Hyku
|