bulkrax 9.4.3 → 9.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/factories/bulkrax/object_factory_interface.rb +18 -1
- data/app/factories/bulkrax/valkyrie_object_factory.rb +9 -8
- data/app/models/bulkrax/csv_entry.rb +8 -9
- data/app/models/concerns/bulkrax/has_matchers.rb +51 -16
- data/app/parsers/concerns/bulkrax/csv_parser/csv_validation.rb +23 -1
- data/app/services/bulkrax/csv_template/column_builder.rb +11 -1
- data/app/services/bulkrax/csv_template/mapping_manager.rb +22 -0
- data/app/services/bulkrax/csv_template/value_determiner.rb +8 -1
- data/lib/bulkrax/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: dd9cb038733c14b67261147e37dfe9087776846d22f327b3985761e58267c129
|
|
4
|
+
data.tar.gz: 46f82669413a304c2c8f511368f7a51e088908bf387fa617234597b26bea07e7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: fe9ef3f4391865d4f95f74fd2eb4f8e95f529963fbbd2d6439f907ace1908a5c70249d59f73c15593f2d237f3f653d945a458e50e3beaf3727928f6529614d76
|
|
7
|
+
data.tar.gz: 4413721e9203032f1bb03add7072ece3c58ec7ec8afcd51b103180b53ca8808df87de0c2793c6101646a8e7285058234cc76628e46c213eda4a62adbacb97b0f
|
|
@@ -468,7 +468,24 @@ module Bulkrax
|
|
|
468
468
|
# Regardless of what the Parser gives us, these are the properties we are
|
|
469
469
|
# prepared to accept.
|
|
470
470
|
def permitted_attributes
|
|
471
|
-
klass.properties.keys.map(&:to_sym) + base_permitted_attributes
|
|
471
|
+
bare = klass.properties.keys.map(&:to_sym) + base_permitted_attributes
|
|
472
|
+
bare + nested_attributes_keys(bare)
|
|
473
|
+
end
|
|
474
|
+
|
|
475
|
+
# Permits `*_attributes` virtual keys when the corresponding bare property
|
|
476
|
+
# is itself permitted. Field mappings declaring `nested_attributes: true`
|
|
477
|
+
# (see Bulkrax::HasMatchers#set_parsed_object_data) emit data under keys
|
|
478
|
+
# like `redirects_attributes`; without this, the slice in
|
|
479
|
+
# #transform_attributes would drop them and downstream form populators
|
|
480
|
+
# would receive nothing.
|
|
481
|
+
def nested_attributes_keys(bare_permitted)
|
|
482
|
+
bare_set = bare_permitted.map(&:to_sym).to_set
|
|
483
|
+
attributes.keys.filter_map do |key|
|
|
484
|
+
key_str = key.to_s
|
|
485
|
+
next unless key_str.end_with?('_attributes')
|
|
486
|
+
bare_name = key_str.sub(/_attributes\z/, '').to_sym
|
|
487
|
+
key.to_sym if bare_set.include?(bare_name)
|
|
488
|
+
end.uniq
|
|
472
489
|
end
|
|
473
490
|
|
|
474
491
|
# Return a copy of the given attributes, such that all values that are empty
|
|
@@ -498,14 +498,15 @@ module Bulkrax
|
|
|
498
498
|
#
|
|
499
499
|
# @return [Array<Symbols>]
|
|
500
500
|
def permitted_attributes
|
|
501
|
-
@permitted_attributes ||=
|
|
502
|
-
base_permitted_attributes + if klass.respond_to?(:schema)
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
501
|
+
@permitted_attributes ||= begin
|
|
502
|
+
bare = base_permitted_attributes + if klass.respond_to?(:schema)
|
|
503
|
+
admin_set_id = attributes[:admin_set_id] || attributes['admin_set_id']
|
|
504
|
+
Bulkrax::ValkyrieObjectFactory.schema_properties(klass: klass, admin_set_id: admin_set_id)
|
|
505
|
+
else
|
|
506
|
+
klass.properties.keys.map(&:to_sym)
|
|
507
|
+
end
|
|
508
|
+
(bare + nested_attributes_keys(bare)).uniq
|
|
509
|
+
end
|
|
509
510
|
end
|
|
510
511
|
|
|
511
512
|
def update_work(attrs)
|
|
@@ -329,15 +329,14 @@ module Bulkrax
|
|
|
329
329
|
end
|
|
330
330
|
|
|
331
331
|
def object_metadata(data)
|
|
332
|
-
#
|
|
333
|
-
#
|
|
334
|
-
#
|
|
335
|
-
#
|
|
336
|
-
#
|
|
337
|
-
#
|
|
338
|
-
#
|
|
339
|
-
|
|
340
|
-
data = data.map { |d| eval(d) }.flatten # rubocop:disable Security/Eval
|
|
332
|
+
# Each `d` may be either a stringified Ruby hash literal (legacy
|
|
333
|
+
# ActiveFedora persistence) or a plain Hash (Valkyrie/Postgres
|
|
334
|
+
# JSONB). For the legacy stringified form we eval to recover the
|
|
335
|
+
# hash; for plain Hashes we pass through. Using eval is a very bad
|
|
336
|
+
# idea as it will execute the value of `d` within the full Ruby
|
|
337
|
+
# interpreter context — only do it when we know the input is a
|
|
338
|
+
# stringified hash.
|
|
339
|
+
data = data.map { |d| d.is_a?(Hash) ? d : eval(d) }.flatten # rubocop:disable Security/Eval
|
|
341
340
|
|
|
342
341
|
data.each_with_index do |obj, index|
|
|
343
342
|
next if obj.nil?
|
|
@@ -38,7 +38,7 @@ module Bulkrax
|
|
|
38
38
|
|
|
39
39
|
if object_name
|
|
40
40
|
Rails.logger.info("Bulkrax Column automatically matched object #{node_name}, #{node_content}")
|
|
41
|
-
|
|
41
|
+
init_object_container(object_name, object_multiple)
|
|
42
42
|
end
|
|
43
43
|
|
|
44
44
|
value = if matcher
|
|
@@ -60,6 +60,30 @@ module Bulkrax
|
|
|
60
60
|
mapping&.[](field)&.[]('object')
|
|
61
61
|
end
|
|
62
62
|
|
|
63
|
+
# When any field-mapping sibling under `object_name` carries
|
|
64
|
+
# `nested_attributes: true`, Bulkrax routes the imported data through
|
|
65
|
+
# `parsed_metadata["#{object_name}_attributes"]` as a numbered-key hash
|
|
66
|
+
# (with `_destroy: 'false'` per row) — the shape consumed by Reform's
|
|
67
|
+
# nested-attributes machinery and other `*_attributes`-style populators.
|
|
68
|
+
def nested_attributes_object?(object_name)
|
|
69
|
+
return false unless mapping.is_a?(Hash) && object_name.present?
|
|
70
|
+
mapping.any? { |_, cfg| cfg.is_a?(Hash) && cfg['object'] == object_name && cfg['nested_attributes'] }
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def parsed_object_target_key(object_name)
|
|
74
|
+
nested_attributes_object?(object_name) ? "#{object_name}_attributes" : object_name
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def init_object_container(object_name, object_multiple)
|
|
78
|
+
target_key = parsed_object_target_key(object_name)
|
|
79
|
+
default = if object_multiple
|
|
80
|
+
nested_attributes_object?(object_name) ? {} : [{}]
|
|
81
|
+
else
|
|
82
|
+
{}
|
|
83
|
+
end
|
|
84
|
+
parsed_metadata[target_key] ||= default
|
|
85
|
+
end
|
|
86
|
+
|
|
63
87
|
def set_parsed_data(name, value)
|
|
64
88
|
return parsed_metadata[name] = value unless multiple?(name)
|
|
65
89
|
|
|
@@ -69,22 +93,33 @@ module Bulkrax
|
|
|
69
93
|
end
|
|
70
94
|
|
|
71
95
|
def set_parsed_object_data(object_multiple, object_name, name, index, value)
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
96
|
+
target_key = parsed_object_target_key(object_name)
|
|
97
|
+
target = object_target_for(target_key, object_name, object_multiple, index)
|
|
98
|
+
assign_object_value(target, name, value)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Resolve the hash slot that `name` should be written into, initializing
|
|
102
|
+
# any intermediate containers. Returns the leaf hash so the caller can
|
|
103
|
+
# assign the value with a single statement.
|
|
104
|
+
def object_target_for(target_key, object_name, object_multiple, index)
|
|
105
|
+
return parsed_metadata[target_key] unless object_multiple
|
|
106
|
+
|
|
107
|
+
idx = index || 0
|
|
108
|
+
if nested_attributes_object?(object_name)
|
|
109
|
+
parsed_metadata[target_key][idx.to_s] ||= { '_destroy' => 'false' }
|
|
110
|
+
parsed_metadata[target_key][idx.to_s]
|
|
81
111
|
else
|
|
82
|
-
parsed_metadata[
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
112
|
+
parsed_metadata[target_key][idx] ||= {}
|
|
113
|
+
parsed_metadata[target_key][idx]
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def assign_object_value(target, name, value)
|
|
118
|
+
target[name] ||= []
|
|
119
|
+
if value.is_a?(Array)
|
|
120
|
+
target[name] += value
|
|
121
|
+
else
|
|
122
|
+
target[name] = value
|
|
88
123
|
end
|
|
89
124
|
end
|
|
90
125
|
|
|
@@ -84,7 +84,15 @@ module Bulkrax
|
|
|
84
84
|
all_models = field_metadata.keys
|
|
85
85
|
valid_headers = build_valid_validation_headers(mapping_manager, field_analyzer,
|
|
86
86
|
all_models, mappings, field_metadata)
|
|
87
|
-
suffixed
|
|
87
|
+
# Only allow a suffixed header (e.g. `creator_1`, `redirect_path_2`)
|
|
88
|
+
# when its base name is itself recognised. The blanket allow that
|
|
89
|
+
# used to live here let through any *_<digits> column, which masked
|
|
90
|
+
# typos in numbered columns at validation time even though the real
|
|
91
|
+
# importer would fail to map them.
|
|
92
|
+
known_property_keys = (field_metadata || {}).values.flat_map { |m| Array(m[:properties]) }.to_set
|
|
93
|
+
suffixed = headers.select do |h|
|
|
94
|
+
h.match?(/_\d+\z/) && header_base_recognized?(h, valid_headers, mapping_manager, known_property_keys)
|
|
95
|
+
end
|
|
88
96
|
valid_headers = (valid_headers + suffixed).uniq
|
|
89
97
|
|
|
90
98
|
{
|
|
@@ -96,6 +104,20 @@ module Bulkrax
|
|
|
96
104
|
}
|
|
97
105
|
end
|
|
98
106
|
|
|
107
|
+
# Mirrors the recognition rule used by
|
|
108
|
+
# find_unrecognized_validation_headers: a header's base name is
|
|
109
|
+
# recognised if it appears in valid_headers directly or if its
|
|
110
|
+
# mapping_manager#mapped_to_key resolves to a known model property.
|
|
111
|
+
# known_property_keys is precomputed by check_headers so this can be
|
|
112
|
+
# called per-header without rebuilding the set each time.
|
|
113
|
+
def header_base_recognized?(header, valid_headers, mapping_manager, known_property_keys)
|
|
114
|
+
base = header.sub(/_\d+\z/, '')
|
|
115
|
+
return true if valid_headers.include?(base)
|
|
116
|
+
|
|
117
|
+
mapped_key = mapping_manager&.mapped_to_key(base)
|
|
118
|
+
mapped_key.present? && known_property_keys.include?(mapped_key)
|
|
119
|
+
end
|
|
120
|
+
|
|
99
121
|
def extract_hierarchy_items(csv_data, all_ids, find_record, mappings)
|
|
100
122
|
extract_validation_items(
|
|
101
123
|
csv_data, all_ids, find_record,
|
|
@@ -35,12 +35,22 @@ module Bulkrax
|
|
|
35
35
|
properties = field_lists
|
|
36
36
|
.flat_map { |item| item.values.flat_map { |config| config["properties"] || [] } }
|
|
37
37
|
.uniq
|
|
38
|
-
.
|
|
38
|
+
.flat_map { |property| columns_for_property(property) }
|
|
39
39
|
.uniq
|
|
40
40
|
|
|
41
41
|
(properties - required_columns).sort
|
|
42
42
|
end
|
|
43
43
|
|
|
44
|
+
# When a property is the target of one or more `object:` field mappings,
|
|
45
|
+
# emit each of those mappings' `from:` columns (e.g. redirect_path,
|
|
46
|
+
# redirect_canonical, redirect_sequence) rather than the bare property
|
|
47
|
+
# name (redirects). Otherwise fall back to the standard 1:1 mapping.
|
|
48
|
+
def columns_for_property(property)
|
|
49
|
+
nested = @service.mapping_manager.object_columns_for(property)
|
|
50
|
+
return nested if nested.any?
|
|
51
|
+
[@service.mapping_manager.key_to_mapped_column(property)]
|
|
52
|
+
end
|
|
53
|
+
|
|
44
54
|
def relationship_columns
|
|
45
55
|
[
|
|
46
56
|
@service.mapping_manager.find_by_flag("related_children_field_mapping", 'children'),
|
|
@@ -26,6 +26,28 @@ module Bulkrax
|
|
|
26
26
|
@mappings.dig(key, "from")&.first || key
|
|
27
27
|
end
|
|
28
28
|
|
|
29
|
+
# Returns the `object:` value for a given mapping key, or nil. Mirrors
|
|
30
|
+
# the importer-side `Bulkrax::HasMatchers#get_object_name` for callers
|
|
31
|
+
# working with the template-side mapping manager.
|
|
32
|
+
def get_object_name(key)
|
|
33
|
+
@mappings.dig(key, "object")
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Returns the column names that target a given object name via the
|
|
37
|
+
# `object:` field-mapping pattern. The template generator uses this to
|
|
38
|
+
# emit the per-child columns (e.g. redirect_path, redirect_canonical,
|
|
39
|
+
# redirect_sequence) instead of the bare property name (redirects).
|
|
40
|
+
# Numbering is intentionally omitted — the template shows the column
|
|
41
|
+
# shape once; CSV rows can repeat the column with numeric suffixes
|
|
42
|
+
# (e.g. redirect_path_1, redirect_path_2) at import time.
|
|
43
|
+
def object_columns_for(object_name)
|
|
44
|
+
@mappings
|
|
45
|
+
.select { |_k, v| v.is_a?(Hash) && v["object"] == object_name }
|
|
46
|
+
.values
|
|
47
|
+
.flat_map { |v| Array(v["from"]) }
|
|
48
|
+
.uniq
|
|
49
|
+
end
|
|
50
|
+
|
|
29
51
|
def find_by_flag(field_name, default)
|
|
30
52
|
@mappings.find { |_k, v| v[field_name] == true }&.first || default
|
|
31
53
|
end
|
|
@@ -12,9 +12,16 @@ module Bulkrax
|
|
|
12
12
|
def determine_value(column, model_name, field_list)
|
|
13
13
|
key = @service.mapping_manager.mapped_to_key(column)
|
|
14
14
|
required_terms = field_list.dig(model_name, 'required_terms')
|
|
15
|
+
properties = field_list.dig(model_name, "properties") || []
|
|
16
|
+
object_name = @service.mapping_manager.get_object_name(key)
|
|
15
17
|
|
|
16
|
-
if
|
|
18
|
+
if properties.include?(key)
|
|
17
19
|
mark_required_or_optional(key, required_terms)
|
|
20
|
+
elsif object_name && properties.include?(object_name)
|
|
21
|
+
# Column belongs to an `object:` mapping (e.g. `redirect_path` → object
|
|
22
|
+
# `redirects`). Treat the column as required/optional based on the
|
|
23
|
+
# parent property's required-terms list.
|
|
24
|
+
mark_required_or_optional(object_name, required_terms)
|
|
18
25
|
elsif special_column?(column, key)
|
|
19
26
|
special_value(column, key, model_name, required_terms)
|
|
20
27
|
end
|
data/lib/bulkrax/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: bulkrax
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 9.
|
|
4
|
+
version: 9.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Rob Kaufman
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-05-08 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rails
|