bulkrax 9.3.5 → 9.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +11 -1
- data/app/assets/javascripts/bulkrax/application.js +2 -1
- data/app/assets/javascripts/bulkrax/bulkrax.js +13 -4
- data/app/assets/javascripts/bulkrax/bulkrax_utils.js +96 -0
- data/app/assets/javascripts/bulkrax/datatables.js +1 -0
- data/app/assets/javascripts/bulkrax/entries.js +17 -10
- data/app/assets/javascripts/bulkrax/importers.js.erb +9 -2
- data/app/assets/javascripts/bulkrax/importers_stepper.js +2420 -0
- data/app/assets/stylesheets/bulkrax/application.css +1 -1
- data/app/assets/stylesheets/bulkrax/stepper/_header.scss +83 -0
- data/app/assets/stylesheets/bulkrax/stepper/_mixins.scss +26 -0
- data/app/assets/stylesheets/bulkrax/stepper/_navigation.scss +103 -0
- data/app/assets/stylesheets/bulkrax/stepper/_responsive.scss +46 -0
- data/app/assets/stylesheets/bulkrax/stepper/_review.scss +92 -0
- data/app/assets/stylesheets/bulkrax/stepper/_settings.scss +106 -0
- data/app/assets/stylesheets/bulkrax/stepper/_success.scss +26 -0
- data/app/assets/stylesheets/bulkrax/stepper/_summary.scss +171 -0
- data/app/assets/stylesheets/bulkrax/stepper/_upload.scss +339 -0
- data/app/assets/stylesheets/bulkrax/stepper/_validation.scss +237 -0
- data/app/assets/stylesheets/bulkrax/stepper/_variables.scss +46 -0
- data/app/assets/stylesheets/bulkrax/stepper.scss +32 -0
- data/app/controllers/bulkrax/guided_imports_controller.rb +175 -0
- data/app/controllers/bulkrax/importers_controller.rb +28 -31
- data/app/controllers/concerns/bulkrax/guided_import_demo_scenarios.rb +201 -0
- data/app/controllers/concerns/bulkrax/importer_file_handler.rb +212 -0
- data/app/errors/bulkrax/unzip_error.rb +16 -0
- data/app/factories/bulkrax/object_factory.rb +3 -2
- data/app/factories/bulkrax/valkyrie_object_factory.rb +61 -17
- data/app/jobs/bulkrax/importer_job.rb +42 -4
- data/app/models/bulkrax/csv_entry.rb +27 -7
- data/app/models/bulkrax/entry.rb +4 -0
- data/app/models/bulkrax/importer.rb +27 -10
- data/app/models/concerns/bulkrax/has_matchers.rb +2 -2
- data/app/models/concerns/bulkrax/importer_exporter_behavior.rb +6 -5
- data/app/parsers/bulkrax/application_parser.rb +63 -20
- data/app/parsers/bulkrax/bagit_parser.rb +12 -0
- data/app/parsers/bulkrax/csv_parser.rb +168 -25
- data/app/parsers/concerns/bulkrax/csv_parser/csv_template_generation.rb +73 -0
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation.rb +133 -0
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_helpers.rb +282 -0
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation_hierarchy.rb +96 -0
- data/app/services/bulkrax/csv_template/column_builder.rb +60 -0
- data/app/services/bulkrax/csv_template/column_descriptor.rb +58 -0
- data/app/services/bulkrax/csv_template/csv_builder.rb +83 -0
- data/app/services/bulkrax/csv_template/explanation_builder.rb +57 -0
- data/app/services/bulkrax/csv_template/field_analyzer.rb +56 -0
- data/app/services/bulkrax/csv_template/file_path_generator.rb +47 -0
- data/app/services/bulkrax/csv_template/file_validator.rb +68 -0
- data/app/services/bulkrax/csv_template/mapping_manager.rb +55 -0
- data/app/services/bulkrax/csv_template/model_loader.rb +50 -0
- data/app/services/bulkrax/csv_template/row_builder.rb +35 -0
- data/app/services/bulkrax/csv_template/schema_analyzer.rb +70 -0
- data/app/services/bulkrax/csv_template/split_formatter.rb +44 -0
- data/app/services/bulkrax/csv_template/value_determiner.rb +68 -0
- data/app/services/bulkrax/stepper_response_formatter.rb +347 -0
- data/app/services/bulkrax/validation_error_csv_builder.rb +99 -0
- data/app/validators/bulkrax/csv_row/child_reference.rb +56 -0
- data/app/validators/bulkrax/csv_row/circular_reference.rb +71 -0
- data/app/validators/bulkrax/csv_row/controlled_vocabulary.rb +74 -0
- data/app/validators/bulkrax/csv_row/duplicate_identifier.rb +63 -0
- data/app/validators/bulkrax/csv_row/missing_source_identifier.rb +31 -0
- data/app/validators/bulkrax/csv_row/parent_reference.rb +59 -0
- data/app/validators/bulkrax/csv_row/required_values.rb +64 -0
- data/app/views/bulkrax/guided_imports/new.html.erb +567 -0
- data/app/views/bulkrax/importers/index.html.erb +6 -1
- data/app/views/bulkrax/importers/new.html.erb +1 -1
- data/app/views/bulkrax/importers/show.html.erb +17 -1
- data/config/i18n-tasks.yml +195 -0
- data/config/locales/bulkrax.de.yml +508 -0
- data/config/locales/bulkrax.en.yml +463 -233
- data/config/locales/bulkrax.es.yml +508 -0
- data/config/locales/bulkrax.fr.yml +508 -0
- data/config/locales/bulkrax.it.yml +508 -0
- data/config/locales/bulkrax.pt-BR.yml +508 -0
- data/config/locales/bulkrax.zh.yml +507 -0
- data/config/routes.rb +10 -1
- data/lib/bulkrax/data/demo_scenarios.json +2235 -0
- data/lib/bulkrax/version.rb +1 -1
- data/lib/bulkrax.rb +31 -0
- metadata +56 -16
- data/app/services/bulkrax/sample_csv_service/column_builder.rb +0 -58
- data/app/services/bulkrax/sample_csv_service/column_descriptor.rb +0 -56
- data/app/services/bulkrax/sample_csv_service/csv_builder.rb +0 -82
- data/app/services/bulkrax/sample_csv_service/explanation_builder.rb +0 -51
- data/app/services/bulkrax/sample_csv_service/field_analyzer.rb +0 -54
- data/app/services/bulkrax/sample_csv_service/file_path_generator.rb +0 -16
- data/app/services/bulkrax/sample_csv_service/mapping_manager.rb +0 -36
- data/app/services/bulkrax/sample_csv_service/model_loader.rb +0 -40
- data/app/services/bulkrax/sample_csv_service/row_builder.rb +0 -33
- data/app/services/bulkrax/sample_csv_service/schema_analyzer.rb +0 -69
- data/app/services/bulkrax/sample_csv_service/split_formatter.rb +0 -42
- data/app/services/bulkrax/sample_csv_service/value_determiner.rb +0 -67
- data/app/services/bulkrax/sample_csv_service.rb +0 -78
- /data/{app/services → lib}/wings/custom_queries/find_by_source_identifier.rb +0 -0
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
module CsvRow
|
|
5
|
+
##
|
|
6
|
+
# Validates that each row has a unique source_identifier.
|
|
7
|
+
# Uses context[:seen_ids] (Hash: id => first_seen_row_number) to detect duplicates.
|
|
8
|
+
module DuplicateIdentifier
|
|
9
|
+
def self.call(record, row_index, context)
|
|
10
|
+
source_id = record[:source_identifier]
|
|
11
|
+
return if source_id.blank? && Bulkrax.fill_in_blank_source_identifiers.present?
|
|
12
|
+
|
|
13
|
+
source_id_label = context[:source_identifier] || 'source_identifier'
|
|
14
|
+
first_row = context[:seen_ids][source_id]
|
|
15
|
+
|
|
16
|
+
if first_row
|
|
17
|
+
add_duplicate_error(context, row_index, source_id, source_id_label, first_row)
|
|
18
|
+
else
|
|
19
|
+
context[:seen_ids][source_id] = row_index
|
|
20
|
+
add_existing_warning(context, row_index, source_id, source_id_label)
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def self.add_duplicate_error(context, row_index, source_id, source_id_label, first_row)
|
|
25
|
+
context[:errors] << {
|
|
26
|
+
row: row_index,
|
|
27
|
+
source_identifier: source_id,
|
|
28
|
+
severity: 'error',
|
|
29
|
+
category: 'duplicate_source_identifier',
|
|
30
|
+
column: source_id_label,
|
|
31
|
+
value: source_id,
|
|
32
|
+
message: I18n.t('bulkrax.importer.guided_import.validation.duplicate_identifier_validator.errors.message',
|
|
33
|
+
value: source_id,
|
|
34
|
+
field: source_id_label,
|
|
35
|
+
original_row: first_row),
|
|
36
|
+
suggestion: I18n.t('bulkrax.importer.guided_import.validation.duplicate_identifier_validator.errors.suggestion',
|
|
37
|
+
field: source_id_label)
|
|
38
|
+
}
|
|
39
|
+
end
|
|
40
|
+
private_class_method :add_duplicate_error
|
|
41
|
+
|
|
42
|
+
def self.add_existing_warning(context, row_index, source_id, source_id_label)
|
|
43
|
+
find_record = context[:find_record_by_source_identifier]
|
|
44
|
+
return unless find_record&.call(source_id)
|
|
45
|
+
|
|
46
|
+
context[:errors] << {
|
|
47
|
+
row: row_index,
|
|
48
|
+
source_identifier: source_id,
|
|
49
|
+
severity: 'warning',
|
|
50
|
+
category: 'existing_source_identifier',
|
|
51
|
+
column: source_id_label,
|
|
52
|
+
value: source_id,
|
|
53
|
+
message: I18n.t('bulkrax.importer.guided_import.validation.existing_source_identifier_validator.warnings.message',
|
|
54
|
+
value: source_id,
|
|
55
|
+
field: source_id_label),
|
|
56
|
+
suggestion: I18n.t('bulkrax.importer.guided_import.validation.existing_source_identifier_validator.warnings.suggestion',
|
|
57
|
+
field: source_id_label)
|
|
58
|
+
}
|
|
59
|
+
end
|
|
60
|
+
private_class_method :add_existing_warning
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
module CsvRow
|
|
5
|
+
##
|
|
6
|
+
# Validates that each row has a value for source_identifier unless
|
|
7
|
+
# fill_in_blank_source_identifiers is configured (in which case Bulkrax
|
|
8
|
+
# will generate one automatically).
|
|
9
|
+
module MissingSourceIdentifier
|
|
10
|
+
def self.call(record, row_index, context)
|
|
11
|
+
return if Bulkrax.fill_in_blank_source_identifiers.present?
|
|
12
|
+
return if record[:source_identifier].present?
|
|
13
|
+
|
|
14
|
+
source_id_label = context[:source_identifier] || 'source_identifier'
|
|
15
|
+
|
|
16
|
+
context[:errors] << {
|
|
17
|
+
row: row_index,
|
|
18
|
+
source_identifier: nil,
|
|
19
|
+
severity: 'error',
|
|
20
|
+
category: 'missing_source_identifier',
|
|
21
|
+
column: source_id_label,
|
|
22
|
+
value: nil,
|
|
23
|
+
message: I18n.t('bulkrax.importer.guided_import.validation.missing_source_identifier_validator.errors.message',
|
|
24
|
+
field: source_id_label),
|
|
25
|
+
suggestion: I18n.t('bulkrax.importer.guided_import.validation.missing_source_identifier_validator.errors.suggestion',
|
|
26
|
+
field: source_id_label)
|
|
27
|
+
}
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
module CsvRow
|
|
5
|
+
##
|
|
6
|
+
# Validates that any parent references in a row point to source identifiers
|
|
7
|
+
# that exist either elsewhere in the same CSV or as existing repository records.
|
|
8
|
+
# Uses context[:all_ids] (Set of all source identifiers) to validate references
|
|
9
|
+
# within the CSV, and context[:find_record_by_source_identifier] (callable) to
|
|
10
|
+
# look up existing records in the same way the importer does at runtime.
|
|
11
|
+
# Uses context[:parent_split_pattern] (String/Regexp, may be nil) for multi-value splitting.
|
|
12
|
+
module ParentReference
|
|
13
|
+
def self.call(record, row_index, context)
|
|
14
|
+
all_ids = context[:all_ids]
|
|
15
|
+
find_record = context[:find_record_by_source_identifier]
|
|
16
|
+
|
|
17
|
+
collect_parent_ids(record, context).each do |parent_id|
|
|
18
|
+
next if all_ids.include?(parent_id)
|
|
19
|
+
next if find_record&.call(parent_id)
|
|
20
|
+
|
|
21
|
+
context[:errors] << {
|
|
22
|
+
row: row_index,
|
|
23
|
+
source_identifier: record[:source_identifier],
|
|
24
|
+
severity: 'error',
|
|
25
|
+
category: 'invalid_parent_reference',
|
|
26
|
+
column: 'parent',
|
|
27
|
+
value: parent_id,
|
|
28
|
+
message: I18n.t('bulkrax.importer.guided_import.validation.parent_reference_validator.errors.message',
|
|
29
|
+
value: parent_id,
|
|
30
|
+
field: 'source_identifier'),
|
|
31
|
+
suggestion: I18n.t('bulkrax.importer.guided_import.validation.parent_reference_validator.errors.suggestion')
|
|
32
|
+
}
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def self.collect_parent_ids(record, context)
|
|
37
|
+
split_pattern = context[:parent_split_pattern]
|
|
38
|
+
parent_column = context[:parent_column] || 'parents'
|
|
39
|
+
|
|
40
|
+
base_ids = if split_pattern
|
|
41
|
+
record[:parent].to_s.split(split_pattern).map(&:strip).reject(&:blank?)
|
|
42
|
+
elsif record[:parent].present?
|
|
43
|
+
[record[:parent].to_s.strip]
|
|
44
|
+
else
|
|
45
|
+
[]
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
suffix_pattern = /\A#{Regexp.escape(parent_column)}_\d+\z/
|
|
49
|
+
suffix_ids = record[:raw_row]
|
|
50
|
+
.select { |k, _| k.to_s.match?(suffix_pattern) }
|
|
51
|
+
.values
|
|
52
|
+
.map(&:to_s).map(&:strip).reject(&:blank?)
|
|
53
|
+
|
|
54
|
+
(base_ids + suffix_ids).uniq
|
|
55
|
+
end
|
|
56
|
+
private_class_method :collect_parent_ids
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bulkrax
|
|
4
|
+
module CsvRow
|
|
5
|
+
##
|
|
6
|
+
# Validates that each row provides a value for every required field of its model.
|
|
7
|
+
# Numeric suffixes on column names are normalised before checking
|
|
8
|
+
# (e.g. 'title_1' satisfies the 'title' requirement).
|
|
9
|
+
module RequiredValues
|
|
10
|
+
def self.call(record, row_index, context)
|
|
11
|
+
field_metadata = context[:field_metadata]
|
|
12
|
+
return if field_metadata.blank?
|
|
13
|
+
|
|
14
|
+
using_default = record[:model].blank?
|
|
15
|
+
model = record[:model].presence || Bulkrax.default_work_type
|
|
16
|
+
metadata = field_metadata[model]
|
|
17
|
+
return if metadata.blank?
|
|
18
|
+
|
|
19
|
+
add_default_work_type_warning(context, record, row_index, model) if using_default
|
|
20
|
+
add_missing_required_value_errors(context, record, row_index, metadata)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def self.add_default_work_type_warning(context, record, row_index, model)
|
|
24
|
+
# Suppress per-row warning when a file-level notice already covers all rows.
|
|
25
|
+
return if context[:notices]&.any? { |n| n[:field] == 'model' }
|
|
26
|
+
|
|
27
|
+
context[:errors] << {
|
|
28
|
+
row: row_index,
|
|
29
|
+
source_identifier: record[:source_identifier],
|
|
30
|
+
severity: 'warning',
|
|
31
|
+
category: 'default_work_type_used',
|
|
32
|
+
column: 'model',
|
|
33
|
+
value: nil,
|
|
34
|
+
message: I18n.t('bulkrax.importer.guided_import.validation.default_work_type_validator.warnings.message',
|
|
35
|
+
default_work_type: model),
|
|
36
|
+
suggestion: I18n.t('bulkrax.importer.guided_import.validation.default_work_type_validator.warnings.suggestion')
|
|
37
|
+
}
|
|
38
|
+
end
|
|
39
|
+
private_class_method :add_default_work_type_warning
|
|
40
|
+
|
|
41
|
+
def self.add_missing_required_value_errors(context, record, row_index, metadata)
|
|
42
|
+
(metadata[:required_terms] || []).each do |field|
|
|
43
|
+
next if record[:raw_row].any? { |key, value| normalize_header(key.to_s) == field && value.present? }
|
|
44
|
+
|
|
45
|
+
context[:errors] << {
|
|
46
|
+
row: row_index,
|
|
47
|
+
source_identifier: record[:source_identifier],
|
|
48
|
+
severity: 'error',
|
|
49
|
+
category: 'missing_required_value',
|
|
50
|
+
column: field,
|
|
51
|
+
value: nil,
|
|
52
|
+
message: I18n.t('bulkrax.importer.guided_import.validation.required_field_validator.errors.message', field: field),
|
|
53
|
+
suggestion: I18n.t('bulkrax.importer.guided_import.validation.required_field_validator.errors.suggestion', field: field)
|
|
54
|
+
}
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
private_class_method :add_missing_required_value_errors
|
|
58
|
+
|
|
59
|
+
def self.normalize_header(header)
|
|
60
|
+
header.sub(/_\d+\z/, '')
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|