typed_eav 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +157 -0
- data/README.md +241 -60
- data/app/models/typed_eav/field/base.rb +77 -27
- data/app/models/typed_eav/field/currency.rb +43 -0
- data/app/models/typed_eav/field/file.rb +1 -1
- data/app/models/typed_eav/field/image.rb +1 -1
- data/app/models/typed_eav/field/reference.rb +11 -0
- data/app/models/typed_eav/section.rb +11 -5
- data/app/models/typed_eav/value.rb +95 -32
- data/db/migrate/20260430000000_add_parent_scope_to_typed_eav_partitions.rb +1 -1
- data/db/migrate/20260712000000_enforce_parent_scope_invariant.rb +4 -0
- data/db/migrate/20260816000000_use_partial_covering_scalar_indexes.rb +198 -0
- data/lib/typed_eav/bulk_read.rb +41 -9
- data/lib/typed_eav/bulk_upsert.rb +141 -0
- data/lib/typed_eav/bulk_write.rb +65 -39
- data/lib/typed_eav/config.rb +19 -21
- data/lib/typed_eav/csv_mapper.rb +1 -1
- data/lib/typed_eav/engine.rb +16 -27
- data/lib/typed_eav/entity_query.rb +32 -7
- data/lib/typed_eav/event_dispatcher.rb +21 -30
- data/lib/typed_eav/field/typed_storage.rb +48 -0
- data/lib/typed_eav/field_deletion.rb +75 -0
- data/lib/typed_eav/query_builder.rb +17 -27
- data/lib/typed_eav/registry.rb +7 -8
- data/lib/typed_eav/version.rb +1 -1
- data/lib/typed_eav/versioning/subscriber.rb +25 -29
- data/lib/typed_eav/versioning.rb +59 -41
- data/lib/typed_eav.rb +2 -0
- metadata +4 -1
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class UsePartialCoveringScalarIndexes < ActiveRecord::Migration[7.1]
|
|
4
|
+
# Concurrent create-before-drop keeps the replacement indexes available
|
|
5
|
+
# throughout the upgrade and leaves rollback able to recreate legacy DDL.
|
|
6
|
+
disable_ddl_transaction!
|
|
7
|
+
|
|
8
|
+
TABLE = :typed_eav_values
|
|
9
|
+
|
|
10
|
+
INDEXES = [
|
|
11
|
+
{
|
|
12
|
+
column: :integer_value,
|
|
13
|
+
replacement: "idx_te_values_field_int_present",
|
|
14
|
+
legacy: "idx_te_values_field_int",
|
|
15
|
+
value_opclass: "int8_ops",
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
column: :decimal_value,
|
|
19
|
+
replacement: "idx_te_values_field_dec_present",
|
|
20
|
+
legacy: "idx_te_values_field_dec",
|
|
21
|
+
value_opclass: "numeric_ops",
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
column: :date_value,
|
|
25
|
+
replacement: "idx_te_values_field_date_present",
|
|
26
|
+
legacy: "idx_te_values_field_date",
|
|
27
|
+
value_opclass: "date_ops",
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
column: :datetime_value,
|
|
31
|
+
replacement: "idx_te_values_field_dt_present",
|
|
32
|
+
legacy: "idx_te_values_field_dt",
|
|
33
|
+
value_opclass: "timestamp_ops",
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
column: :boolean_value,
|
|
37
|
+
replacement: "idx_te_values_field_bool_present",
|
|
38
|
+
legacy: "idx_te_values_field_bool",
|
|
39
|
+
value_opclass: "bool_ops",
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
column: :string_value,
|
|
43
|
+
replacement: "idx_te_values_field_str_present",
|
|
44
|
+
legacy: "idx_te_values_field_str",
|
|
45
|
+
value_opclass: "text_pattern_ops",
|
|
46
|
+
},
|
|
47
|
+
].freeze
|
|
48
|
+
|
|
49
|
+
def up
|
|
50
|
+
INDEXES.each { |definition| ensure_index!(replacement_definition(definition)) }
|
|
51
|
+
assert_all_valid!(INDEXES.map { |definition| replacement_definition(definition) })
|
|
52
|
+
INDEXES.each { |definition| remove_expected_index!(legacy_definition(definition)) }
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def down
|
|
56
|
+
INDEXES.each { |definition| ensure_index!(legacy_definition(definition)) }
|
|
57
|
+
assert_all_valid!(INDEXES.map { |definition| legacy_definition(definition) })
|
|
58
|
+
INDEXES.each { |definition| remove_expected_index!(replacement_definition(definition)) }
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
def replacement_definition(definition)
|
|
64
|
+
definition.merge(
|
|
65
|
+
name: definition.fetch(:replacement),
|
|
66
|
+
include: ["entity_id"],
|
|
67
|
+
predicate: "#{definition.fetch(:column)} IS NOT NULL",
|
|
68
|
+
)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def legacy_definition(definition)
|
|
72
|
+
definition.merge(
|
|
73
|
+
name: definition.fetch(:legacy),
|
|
74
|
+
include: %w[entity_id entity_type],
|
|
75
|
+
predicate: nil,
|
|
76
|
+
)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def ensure_index!(expected)
|
|
80
|
+
actual = catalog_index(expected.fetch(:name))
|
|
81
|
+
return create_and_validate!(expected) unless actual
|
|
82
|
+
return if exact_definition?(actual, expected) && usable?(actual)
|
|
83
|
+
|
|
84
|
+
unless exact_definition?(actual, expected)
|
|
85
|
+
raise ActiveRecord::MigrationError,
|
|
86
|
+
"index #{expected.fetch(:name)} exists with unexpected definition: #{actual.inspect}"
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
say "repairing invalid interrupted index #{expected.fetch(:name)}"
|
|
90
|
+
remove_index TABLE, name: expected.fetch(:name), algorithm: :concurrently
|
|
91
|
+
create_and_validate!(expected)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def create_and_validate!(expected)
|
|
95
|
+
options = {
|
|
96
|
+
name: expected.fetch(:name),
|
|
97
|
+
include: expected.fetch(:include),
|
|
98
|
+
algorithm: :concurrently,
|
|
99
|
+
}
|
|
100
|
+
options[:where] = expected.fetch(:predicate) if expected.fetch(:predicate)
|
|
101
|
+
if expected.fetch(:value_opclass) == "text_pattern_ops"
|
|
102
|
+
options[:opclass] = { expected.fetch(:column) => :text_pattern_ops }
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
add_index TABLE, [:field_id, expected.fetch(:column)], **options
|
|
106
|
+
actual = catalog_index(expected.fetch(:name))
|
|
107
|
+
return if actual && exact_definition?(actual, expected) && usable?(actual)
|
|
108
|
+
|
|
109
|
+
raise ActiveRecord::MigrationError,
|
|
110
|
+
"concurrent index #{expected.fetch(:name)} was not created with its exact valid definition"
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def assert_all_valid!(expected_indexes)
|
|
114
|
+
invalid = expected_indexes.filter_map do |expected|
|
|
115
|
+
actual = catalog_index(expected.fetch(:name))
|
|
116
|
+
expected.fetch(:name) unless actual && exact_definition?(actual, expected) && usable?(actual)
|
|
117
|
+
end
|
|
118
|
+
return if invalid.empty?
|
|
119
|
+
|
|
120
|
+
raise ActiveRecord::MigrationError,
|
|
121
|
+
"refusing to remove existing indexes before all replacements are valid: #{invalid.join(", ")}"
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def remove_expected_index!(expected)
|
|
125
|
+
actual = catalog_index(expected.fetch(:name))
|
|
126
|
+
return unless actual
|
|
127
|
+
|
|
128
|
+
unless exact_definition?(actual, expected)
|
|
129
|
+
raise ActiveRecord::MigrationError,
|
|
130
|
+
"index #{expected.fetch(:name)} exists with unexpected definition: #{actual.inspect}"
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
remove_index TABLE, name: expected.fetch(:name), algorithm: :concurrently
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def usable?(actual)
|
|
137
|
+
actual.fetch("valid") && actual.fetch("ready")
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def exact_definition?(actual, expected)
|
|
141
|
+
actual.fetch("method") == "btree" &&
|
|
142
|
+
!actual.fetch("unique") &&
|
|
143
|
+
actual.fetch("columns") == ["field_id", expected.fetch(:column).to_s, *expected.fetch(:include)] &&
|
|
144
|
+
actual.fetch("key_count") == 2 &&
|
|
145
|
+
actual.fetch("opclasses") == ["int8_ops", expected.fetch(:value_opclass)] &&
|
|
146
|
+
normalized_predicate(actual.fetch("predicate")) == normalized_predicate(expected.fetch(:predicate))
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def normalized_predicate(predicate)
|
|
150
|
+
predicate&.gsub(/[()\s"]/, "")&.downcase
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def catalog_index(name)
|
|
154
|
+
quoted_name = connection.quote(name)
|
|
155
|
+
row = connection.select_one(<<~SQL.squish)
|
|
156
|
+
SELECT
|
|
157
|
+
index_state.indisvalid AS valid,
|
|
158
|
+
index_state.indisready AS ready,
|
|
159
|
+
index_state.indisunique AS unique,
|
|
160
|
+
access_method.amname AS method,
|
|
161
|
+
index_state.indnkeyatts AS key_count,
|
|
162
|
+
pg_get_expr(index_state.indpred, index_state.indrelid) AS predicate,
|
|
163
|
+
array_to_string(ARRAY(
|
|
164
|
+
SELECT attribute.attname
|
|
165
|
+
FROM unnest(index_state.indkey) WITH ORDINALITY AS index_column(attnum, position)
|
|
166
|
+
JOIN pg_attribute attribute
|
|
167
|
+
ON attribute.attrelid = index_state.indrelid
|
|
168
|
+
AND attribute.attnum = index_column.attnum
|
|
169
|
+
ORDER BY index_column.position
|
|
170
|
+
), ',') AS columns,
|
|
171
|
+
array_to_string(ARRAY(
|
|
172
|
+
SELECT operator_class.opcname
|
|
173
|
+
FROM unnest(index_state.indclass) WITH ORDINALITY AS index_opclass(opcoid, position)
|
|
174
|
+
JOIN pg_opclass operator_class ON operator_class.oid = index_opclass.opcoid
|
|
175
|
+
WHERE index_opclass.position <= index_state.indnkeyatts
|
|
176
|
+
ORDER BY index_opclass.position
|
|
177
|
+
), ',') AS opclasses
|
|
178
|
+
FROM pg_class index_relation
|
|
179
|
+
JOIN pg_index index_state ON index_state.indexrelid = index_relation.oid
|
|
180
|
+
JOIN pg_class table_relation ON table_relation.oid = index_state.indrelid
|
|
181
|
+
JOIN pg_namespace namespace ON namespace.oid = table_relation.relnamespace
|
|
182
|
+
JOIN pg_am access_method ON access_method.oid = index_relation.relam
|
|
183
|
+
WHERE namespace.nspname = ANY (current_schemas(false))
|
|
184
|
+
AND table_relation.relname = 'typed_eav_values'
|
|
185
|
+
AND index_relation.relname = #{quoted_name}
|
|
186
|
+
SQL
|
|
187
|
+
return unless row
|
|
188
|
+
|
|
189
|
+
row.merge(
|
|
190
|
+
"valid" => ActiveModel::Type::Boolean.new.cast(row.fetch("valid")),
|
|
191
|
+
"ready" => ActiveModel::Type::Boolean.new.cast(row.fetch("ready")),
|
|
192
|
+
"unique" => ActiveModel::Type::Boolean.new.cast(row.fetch("unique")),
|
|
193
|
+
"key_count" => row.fetch("key_count").to_i,
|
|
194
|
+
"columns" => row.fetch("columns").split(","),
|
|
195
|
+
"opclasses" => row.fetch("opclasses").split(","),
|
|
196
|
+
)
|
|
197
|
+
end
|
|
198
|
+
end
|
data/lib/typed_eav/bulk_read.rb
CHANGED
|
@@ -6,13 +6,12 @@ module TypedEAV
|
|
|
6
6
|
# `HasTypedEAV::InstanceMethods#typed_eav_hash`. N+1-free regardless of
|
|
7
7
|
# record count or field count.
|
|
8
8
|
#
|
|
9
|
-
# ## Pipeline (one
|
|
9
|
+
# ## Pipeline (one batched definition query + one bulk value preload)
|
|
10
10
|
#
|
|
11
11
|
# 1. validate_records! — nil -> ArgumentError; single-class invariant
|
|
12
12
|
# 2. group_by_tuple — `[typed_eav_scope, typed_eav_parent_scope]`
|
|
13
|
-
# 3. winning_ids_by_tuple —
|
|
14
|
-
# `Partition.definitions_by_name`
|
|
15
|
-
# collision-precedence helper)
|
|
13
|
+
# 3. winning_ids_by_tuple — one exact-partition definition query, then
|
|
14
|
+
# `Partition.definitions_by_name` per tuple
|
|
16
15
|
# 4. preload_values — single SELECT across ALL records
|
|
17
16
|
# 5. build_result_hash — per-record inner hash; orphan-skip + winning-id
|
|
18
17
|
# precedence mirrored from the instance path.
|
|
@@ -21,9 +20,9 @@ module TypedEAV
|
|
|
21
20
|
#
|
|
22
21
|
# - 1 SELECT typed_eav_values WHERE entity_type=? AND entity_id IN (?)
|
|
23
22
|
# - 1 SELECT typed_eav_fields WHERE id IN (?) (via includes)
|
|
24
|
-
# - 1 SELECT typed_eav_fields
|
|
23
|
+
# - 1 SELECT typed_eav_fields for the union of requested partitions
|
|
25
24
|
#
|
|
26
|
-
# Total:
|
|
25
|
+
# Total: 3 queries — independent of record count and partition cardinality.
|
|
27
26
|
#
|
|
28
27
|
# ## Single-class invariant
|
|
29
28
|
#
|
|
@@ -74,10 +73,43 @@ module TypedEAV
|
|
|
74
73
|
end
|
|
75
74
|
|
|
76
75
|
def winning_ids_by_tuple(tuples)
|
|
77
|
-
tuples.
|
|
78
|
-
|
|
79
|
-
memo[[s, ps]] = TypedEAV::Partition.definitions_by_name(defs).transform_values(&:id)
|
|
76
|
+
definitions_by_tuple = batched_definitions(tuples).group_by do |definition|
|
|
77
|
+
[definition.scope, definition.parent_scope]
|
|
80
78
|
end
|
|
79
|
+
|
|
80
|
+
tuples.to_h do |tuple|
|
|
81
|
+
s, ps = tuple
|
|
82
|
+
candidate_tuples = [[nil, nil], [s, nil], [s, ps]].uniq
|
|
83
|
+
visible = candidate_tuples.flat_map { |candidate| definitions_by_tuple.fetch(candidate, []) }
|
|
84
|
+
[tuple, TypedEAV::Partition.definitions_by_name(visible).transform_values(&:id)]
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def batched_definitions(tuples)
|
|
89
|
+
tuples.each { |scope, parent_scope| validate_tuple!(scope, parent_scope) }
|
|
90
|
+
relation = TypedEAV::Field::Base.where(entity_type: host_class.name)
|
|
91
|
+
requested = tuples.map { |scope, parent_scope| { "scope" => scope, "parent_scope" => parent_scope } }
|
|
92
|
+
relation.where(<<~SQL.squish, requested.to_json).to_a
|
|
93
|
+
typed_eav_fields.scope IS NULL AND typed_eav_fields.parent_scope IS NULL
|
|
94
|
+
OR EXISTS (
|
|
95
|
+
SELECT 1
|
|
96
|
+
FROM jsonb_to_recordset(?::jsonb) AS requested(scope text, parent_scope text)
|
|
97
|
+
WHERE (
|
|
98
|
+
typed_eav_fields.scope IS NOT DISTINCT FROM requested.scope
|
|
99
|
+
AND typed_eav_fields.parent_scope IS NOT DISTINCT FROM requested.parent_scope
|
|
100
|
+
)
|
|
101
|
+
OR (
|
|
102
|
+
typed_eav_fields.scope IS NOT DISTINCT FROM requested.scope
|
|
103
|
+
AND typed_eav_fields.parent_scope IS NULL
|
|
104
|
+
)
|
|
105
|
+
)
|
|
106
|
+
SQL
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def validate_tuple!(scope, parent_scope)
|
|
110
|
+
return if TypedEAV::ScopeTuple.invariant_satisfied?(scope, parent_scope)
|
|
111
|
+
|
|
112
|
+
raise ArgumentError, "parent_scope cannot be set when scope is blank"
|
|
81
113
|
end
|
|
82
114
|
|
|
83
115
|
def preload_values(records)
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module TypedEAV
|
|
4
|
+
# Reduced-semantics, SQL-oriented bulk value writer.
|
|
5
|
+
#
|
|
6
|
+
# This API runs casting/domain/entity/partition validation and validation
|
|
7
|
+
# callbacks, but does not run host callbacks/validations or host saves, Value
|
|
8
|
+
# persistence callbacks, versioning, or per-record error isolation. Callers must
|
|
9
|
+
# acknowledge those semantics explicitly. It pre-casts and validates every
|
|
10
|
+
# row in a transaction unit before issuing one upsert batch on the host
|
|
11
|
+
# model's connection. Scope resolution and field casting remain authoritative.
|
|
12
|
+
module BulkUpsert
|
|
13
|
+
TYPE_COLUMNS = %i[
|
|
14
|
+
string_value text_value boolean_value integer_value decimal_value date_value datetime_value json_value
|
|
15
|
+
].freeze
|
|
16
|
+
|
|
17
|
+
class << self
|
|
18
|
+
# rubocop:disable Metrics/ParameterLists -- transaction controls are explicit API contract.
|
|
19
|
+
def execute(
|
|
20
|
+
host_class:, records:, values_by_field_name:, acknowledge_reduced_semantics:, transaction: :all, chunk_size: nil
|
|
21
|
+
)
|
|
22
|
+
validate_options!(records, values_by_field_name, acknowledge_reduced_semantics, transaction, chunk_size)
|
|
23
|
+
records = records.to_a
|
|
24
|
+
return 0 if records.empty?
|
|
25
|
+
|
|
26
|
+
validate_record_classes!(host_class, records)
|
|
27
|
+
reject_duplicate_or_unpersisted!(records)
|
|
28
|
+
ensure_connection_owner!(host_class)
|
|
29
|
+
values_by_field_name = normalize_values!(values_by_field_name)
|
|
30
|
+
units = transaction == :all ? [records] : records.each_slice(chunk_size)
|
|
31
|
+
units.sum { |unit| write_unit(host_class, unit, values_by_field_name) }
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
|
|
36
|
+
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- tuple grouping, validation, and row construction are one durability unit.
|
|
37
|
+
def write_unit(host_class, records, values_by_field_name)
|
|
38
|
+
tuples = records.map { |record| [record.typed_eav_scope, record.typed_eav_parent_scope] }.uniq
|
|
39
|
+
fields_by_tuple = tuples.to_h do |tuple|
|
|
40
|
+
# Use the exact tuple at the partition layer. EntityQuery treats
|
|
41
|
+
# explicit scope kwargs as ALL_SCOPES inside TypedEAV.unscoped;
|
|
42
|
+
# that administrative bypass must not cross-contaminate tenants.
|
|
43
|
+
fields = TypedEAV::Partition.definitions_by_name(
|
|
44
|
+
TypedEAV::Partition.visible_fields(
|
|
45
|
+
entity_type: host_class.polymorphic_name,
|
|
46
|
+
scope: tuple[0],
|
|
47
|
+
parent_scope: tuple[1],
|
|
48
|
+
),
|
|
49
|
+
)
|
|
50
|
+
[tuple, fields]
|
|
51
|
+
end
|
|
52
|
+
rows = records.flat_map do |record|
|
|
53
|
+
fields = fields_by_tuple.fetch([record.typed_eav_scope, record.typed_eav_parent_scope])
|
|
54
|
+
values_by_field_name.filter_map do |name, raw|
|
|
55
|
+
field = fields[name.to_s]
|
|
56
|
+
raise ArgumentError, "bulk_upsert field is not visible for #{record.class.name}: #{name}" unless field
|
|
57
|
+
|
|
58
|
+
build_row(host_class, record, field, raw)
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
return 0 if rows.empty?
|
|
62
|
+
|
|
63
|
+
host_class.transaction do
|
|
64
|
+
# rubocop:disable Rails/SkipsModelValidations -- every row is prevalidated in :bulk_upsert context.
|
|
65
|
+
TypedEAV::Value.upsert_all(
|
|
66
|
+
rows,
|
|
67
|
+
unique_by: :idx_te_values_entity_field,
|
|
68
|
+
update_only: (TYPE_COLUMNS + [:updated_at]),
|
|
69
|
+
record_timestamps: false,
|
|
70
|
+
)
|
|
71
|
+
# rubocop:enable Rails/SkipsModelValidations
|
|
72
|
+
end
|
|
73
|
+
rows.length
|
|
74
|
+
end
|
|
75
|
+
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
|
76
|
+
|
|
77
|
+
def build_row(host_class, record, field, raw)
|
|
78
|
+
value = TypedEAV::Value.new(entity: record, field: field)
|
|
79
|
+
value.value = raw
|
|
80
|
+
value.valid?(:bulk_upsert)
|
|
81
|
+
errors = value.errors.to_hash
|
|
82
|
+
raise ArgumentError, "bulk_upsert validation failed for #{field.name}: #{errors.inspect}" if errors.any?
|
|
83
|
+
|
|
84
|
+
now = Time.current
|
|
85
|
+
row = {
|
|
86
|
+
entity_type: host_class.polymorphic_name,
|
|
87
|
+
entity_id: record.id,
|
|
88
|
+
field_id: field.id,
|
|
89
|
+
created_at: now,
|
|
90
|
+
updated_at: now,
|
|
91
|
+
}
|
|
92
|
+
TYPE_COLUMNS.each { |column| row[column] = value[column] }
|
|
93
|
+
row
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def validate_options!(records, values, acknowledged, transaction, chunk_size)
|
|
97
|
+
raise ArgumentError, "bulk_upsert requires an Enumerable of records" if records.nil?
|
|
98
|
+
raise ArgumentError, "bulk_upsert requires a Hash of values_by_field_name" unless values.is_a?(Hash)
|
|
99
|
+
raise ArgumentError, "bulk_upsert requires acknowledge_reduced_semantics: true" unless acknowledged == true
|
|
100
|
+
return if transaction == :all
|
|
101
|
+
return if transaction == :chunks && chunk_size.is_a?(Integer) && chunk_size.positive?
|
|
102
|
+
|
|
103
|
+
raise ArgumentError, "transaction must be :all or :chunks with a positive chunk_size"
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def normalize_values!(values)
|
|
107
|
+
normalized = {}
|
|
108
|
+
values.each do |name, raw|
|
|
109
|
+
key = name.to_s
|
|
110
|
+
if normalized.key?(key)
|
|
111
|
+
raise ArgumentError, "bulk_upsert field keys collide after string normalization: #{key}"
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
normalized[key] = raw
|
|
115
|
+
end
|
|
116
|
+
normalized
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def validate_record_classes!(host_class, records)
|
|
120
|
+
return if records.all?(host_class)
|
|
121
|
+
|
|
122
|
+
raise ArgumentError, "bulk_upsert expects records of class #{host_class.name}"
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def reject_duplicate_or_unpersisted!(records)
|
|
126
|
+
raise ArgumentError, "bulk_upsert requires persisted records" if records.any? { |record| !record.persisted? }
|
|
127
|
+
return if records.map(&:id).uniq.length == records.length
|
|
128
|
+
|
|
129
|
+
raise ArgumentError, "bulk_upsert does not accept duplicate entity records"
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def ensure_connection_owner!(host_class)
|
|
133
|
+
pools = [host_class.connection_pool, TypedEAV::Value.connection_pool, TypedEAV::Field::Base.connection_pool]
|
|
134
|
+
return if pools.map(&:object_id).uniq.one?
|
|
135
|
+
|
|
136
|
+
raise ArgumentError, "bulk_upsert requires host, value, and field connections to share a pool"
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
# rubocop:enable Metrics/ParameterLists
|
|
140
|
+
end
|
|
141
|
+
end
|
data/lib/typed_eav/bulk_write.rb
CHANGED
|
@@ -16,7 +16,8 @@ module TypedEAV
|
|
|
16
16
|
# allocate field UUIDs, and then hand off to `execute_pairs(pairs,
|
|
17
17
|
# effective_grouping, field_uuids)` — a single shared loop that takes
|
|
18
18
|
# ordered `[record, vbn]` pairs and runs the outer-transaction-plus-
|
|
19
|
-
# savepoint-per-record envelope
|
|
19
|
+
# savepoint-per-record envelope; `transaction: :chunks` repeats that
|
|
20
|
+
# envelope per chunk so completed chunks can commit independently.
|
|
20
21
|
#
|
|
21
22
|
# Pair-shaped (not Hash-shaped) so `execute`'s `[record, vbn]` list can
|
|
22
23
|
# carry duplicate in-memory instances of the same persisted row without
|
|
@@ -24,19 +25,25 @@ module TypedEAV
|
|
|
24
25
|
# `execute`'s byte-for-byte behavior contract.
|
|
25
26
|
module BulkWrite
|
|
26
27
|
class << self
|
|
27
|
-
|
|
28
|
-
|
|
28
|
+
# rubocop:disable Metrics/ParameterLists -- transaction controls are explicit API contract.
|
|
29
|
+
def execute(
|
|
30
|
+
host_class:, records:, values_by_field_name:, version_grouping: :default, transaction: :all, chunk_size: nil
|
|
31
|
+
)
|
|
32
|
+
validate_inputs!(records, values_by_field_name, version_grouping, transaction, chunk_size)
|
|
29
33
|
|
|
30
34
|
records = records.to_a
|
|
31
35
|
return { successes: [], errors_by_record: {} } if records.empty?
|
|
32
36
|
|
|
33
37
|
validate_record_classes!(host_class, records)
|
|
38
|
+
ensure_connection_owner!(host_class)
|
|
34
39
|
|
|
35
40
|
effective_grouping = resolve_grouping(version_grouping)
|
|
36
41
|
vbn = values_by_field_name.transform_keys(&:to_s)
|
|
37
42
|
field_uuids = effective_grouping == :per_field ? vbn.keys.index_with { SecureRandom.uuid } : nil
|
|
38
43
|
|
|
39
|
-
execute_pairs(
|
|
44
|
+
execute_pairs(
|
|
45
|
+
host_class, records.map { |r| [r, vbn] }, effective_grouping, field_uuids, transaction, chunk_size
|
|
46
|
+
)
|
|
40
47
|
end
|
|
41
48
|
|
|
42
49
|
# Per-record-varying sibling to `execute`. Accepts a `Hash<host_record,
|
|
@@ -45,12 +52,15 @@ module TypedEAV
|
|
|
45
52
|
#
|
|
46
53
|
# Empty `values_by_record` short-circuits to the empty result without
|
|
47
54
|
# opening a transaction (matches `execute`'s empty-records contract).
|
|
48
|
-
def execute_per_record(
|
|
49
|
-
|
|
55
|
+
def execute_per_record(
|
|
56
|
+
host_class:, values_by_record:, version_grouping: :default, transaction: :all, chunk_size: nil
|
|
57
|
+
)
|
|
58
|
+
validate_per_record_inputs!(values_by_record, version_grouping, transaction, chunk_size)
|
|
50
59
|
|
|
51
60
|
return { successes: [], errors_by_record: {} } if values_by_record.empty?
|
|
52
61
|
|
|
53
62
|
validate_record_classes!(host_class, values_by_record.keys, method: :bulk_set_typed_eav_values_per_record)
|
|
63
|
+
ensure_connection_owner!(host_class)
|
|
54
64
|
|
|
55
65
|
effective_grouping = resolve_grouping(version_grouping)
|
|
56
66
|
pairs = values_by_record.map { |record, vbn| [record, vbn.transform_keys(&:to_s)] }
|
|
@@ -58,15 +68,11 @@ module TypedEAV
|
|
|
58
68
|
pairs.flat_map { |(_record, vbn)| vbn.keys }.uniq.index_with { SecureRandom.uuid }
|
|
59
69
|
end
|
|
60
70
|
|
|
61
|
-
execute_pairs(pairs, effective_grouping, field_uuids)
|
|
71
|
+
execute_pairs(host_class, pairs, effective_grouping, field_uuids, transaction, chunk_size)
|
|
62
72
|
end
|
|
73
|
+
# rubocop:enable Metrics/ParameterLists
|
|
63
74
|
|
|
64
75
|
def apply_record_save(record:, vbn:, effective_grouping:, uuids:, accumulator:)
|
|
65
|
-
push_uuid = case effective_grouping
|
|
66
|
-
when :per_record then uuids[:record]
|
|
67
|
-
when :per_field then uuids[:field].values.first
|
|
68
|
-
end
|
|
69
|
-
|
|
70
76
|
do_save = lambda do
|
|
71
77
|
record.typed_eav_attributes = vbn.map { |name, value| typed_eav_entry_for(name, value) }
|
|
72
78
|
stamp_pending_version_group_ids(record, effective_grouping, uuids)
|
|
@@ -79,11 +85,7 @@ module TypedEAV
|
|
|
79
85
|
end
|
|
80
86
|
end
|
|
81
87
|
|
|
82
|
-
|
|
83
|
-
TypedEAV.with_context(version_group_id: push_uuid, &do_save)
|
|
84
|
-
else
|
|
85
|
-
do_save.call
|
|
86
|
-
end
|
|
88
|
+
do_save.call
|
|
87
89
|
end
|
|
88
90
|
|
|
89
91
|
private
|
|
@@ -97,26 +99,31 @@ module TypedEAV
|
|
|
97
99
|
# the same persisted row iterate each instance separately — matters
|
|
98
100
|
# for `execute`'s byte-for-byte behavior contract on
|
|
99
101
|
# `[Entity.find(1), Entity.find(1)]`-shaped input.
|
|
100
|
-
|
|
102
|
+
# rubocop:disable Metrics/MethodLength -- transaction wrapper retains one readable semantic loop.
|
|
103
|
+
# rubocop:disable Metrics/ParameterLists -- internal transaction controls stay explicit.
|
|
104
|
+
def execute_pairs(host_class, pairs, effective_grouping, field_uuids, transaction, chunk_size)
|
|
101
105
|
successes = []
|
|
102
106
|
errors_by_record = {}
|
|
103
107
|
|
|
104
108
|
with_bulk_definitions_memo do
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
109
|
+
host_class.cache do
|
|
110
|
+
units = transaction == :all ? [pairs] : pairs.each_slice(chunk_size)
|
|
111
|
+
units.each do |unit|
|
|
112
|
+
host_class.transaction do
|
|
113
|
+
unit.each do |(record, vbn)|
|
|
114
|
+
record_uuid = effective_grouping == :per_record ? SecureRandom.uuid : nil
|
|
115
|
+
record_field_uuids = record_scoped_field_uuids(field_uuids, vbn)
|
|
116
|
+
|
|
117
|
+
with_record_scope(record) do
|
|
118
|
+
host_class.transaction(requires_new: true) do
|
|
119
|
+
apply_record_save(
|
|
120
|
+
record: record,
|
|
121
|
+
vbn: vbn,
|
|
122
|
+
effective_grouping: effective_grouping,
|
|
123
|
+
uuids: { record: record_uuid, field: record_field_uuids },
|
|
124
|
+
accumulator: { successes: successes, errors_by_record: errors_by_record },
|
|
125
|
+
)
|
|
126
|
+
end
|
|
120
127
|
end
|
|
121
128
|
end
|
|
122
129
|
end
|
|
@@ -126,6 +133,8 @@ module TypedEAV
|
|
|
126
133
|
|
|
127
134
|
{ successes: successes, errors_by_record: errors_by_record }
|
|
128
135
|
end
|
|
136
|
+
# rubocop:enable Metrics/ParameterLists
|
|
137
|
+
# rubocop:enable Metrics/MethodLength
|
|
129
138
|
|
|
130
139
|
# For `:per_field`, the global `field_uuids` map covers the union of
|
|
131
140
|
# field names across all records' value hashes. The per-record save
|
|
@@ -184,7 +193,7 @@ module TypedEAV
|
|
|
184
193
|
ActiveRecord::Type::Boolean.new.cast(flag)
|
|
185
194
|
end
|
|
186
195
|
|
|
187
|
-
def validate_inputs!(records, values_by_field_name, version_grouping)
|
|
196
|
+
def validate_inputs!(records, values_by_field_name, version_grouping, transaction, chunk_size)
|
|
188
197
|
if records.nil?
|
|
189
198
|
raise ArgumentError,
|
|
190
199
|
"bulk_set_typed_eav_values requires an Enumerable of records, got nil"
|
|
@@ -197,9 +206,17 @@ module TypedEAV
|
|
|
197
206
|
end
|
|
198
207
|
|
|
199
208
|
validate_grouping!(version_grouping)
|
|
209
|
+
validate_transaction!(transaction, chunk_size)
|
|
200
210
|
end
|
|
201
211
|
|
|
202
|
-
def
|
|
212
|
+
def ensure_connection_owner!(host_class)
|
|
213
|
+
pools = [host_class.connection_pool, TypedEAV::Value.connection_pool, TypedEAV::Field::Base.connection_pool]
|
|
214
|
+
return if pools.map(&:object_id).uniq.one?
|
|
215
|
+
|
|
216
|
+
raise ArgumentError, "bulk_write requires host, value, and field connections to share a pool"
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def validate_per_record_inputs!(values_by_record, version_grouping, transaction, chunk_size)
|
|
203
220
|
unless values_by_record.is_a?(Hash)
|
|
204
221
|
raise ArgumentError,
|
|
205
222
|
"bulk_set_typed_eav_values_per_record requires a Hash of values_by_record, " \
|
|
@@ -215,6 +232,14 @@ module TypedEAV
|
|
|
215
232
|
end
|
|
216
233
|
|
|
217
234
|
validate_grouping!(version_grouping)
|
|
235
|
+
validate_transaction!(transaction, chunk_size)
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def validate_transaction!(transaction, chunk_size)
|
|
239
|
+
return if transaction == :all
|
|
240
|
+
return if transaction == :chunks && chunk_size.is_a?(Integer) && chunk_size.positive?
|
|
241
|
+
|
|
242
|
+
raise ArgumentError, "transaction must be :all or :chunks with a positive chunk_size"
|
|
218
243
|
end
|
|
219
244
|
|
|
220
245
|
def validate_grouping!(version_grouping)
|
|
@@ -225,11 +250,12 @@ module TypedEAV
|
|
|
225
250
|
"Supported values: #{valid_grouping.map(&:inspect).join(", ")}."
|
|
226
251
|
end
|
|
227
252
|
|
|
228
|
-
return unless %i[per_record per_field].include?(version_grouping) &&
|
|
253
|
+
return unless %i[per_record per_field].include?(version_grouping) &&
|
|
254
|
+
!TypedEAV::Versioning.atomic_callbacks_installed?
|
|
229
255
|
|
|
230
256
|
raise ArgumentError,
|
|
231
|
-
"version_grouping: #{version_grouping.inspect} was passed but versioning is disabled. " \
|
|
232
|
-
"
|
|
257
|
+
"version_grouping: #{version_grouping.inspect} was passed but versioning is disabled or not installed. " \
|
|
258
|
+
"Enable versioning at boot, or pass " \
|
|
233
259
|
"version_grouping: :none to opt out explicitly, or omit the kwarg to silently no-op."
|
|
234
260
|
end
|
|
235
261
|
|
|
@@ -244,7 +270,7 @@ module TypedEAV
|
|
|
244
270
|
|
|
245
271
|
def resolve_grouping(version_grouping)
|
|
246
272
|
if version_grouping == :default
|
|
247
|
-
TypedEAV.
|
|
273
|
+
TypedEAV::Versioning.atomic_callbacks_installed? ? :per_record : :none
|
|
248
274
|
else
|
|
249
275
|
version_grouping
|
|
250
276
|
end
|