typed_eav 0.5.0 → 0.7.0

data/db/migrate/20260712000000_enforce_parent_scope_invariant.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+class EnforceParentScopeInvariant < ActiveRecord::Migration[7.1]
+  # Strong Migrations requires check-constraint validation outside the default
+  # transaction so the catalog-only add and validation can proceed safely.
+  disable_ddl_transaction!
+
+  CONSTRAINTS = {
+    typed_eav_fields: :chk_te_fields_parent_scope_requires_scope,
+    typed_eav_sections: :chk_te_sections_parent_scope_requires_scope,
+  }.freeze
+
+  INVARIANT_SQL = <<~SQL.squish.freeze
+    parent_scope IS NULL
+    OR BTRIM(parent_scope) = ''
+    OR (scope IS NOT NULL AND BTRIM(scope) <> '')
+  SQL
+
+  ORPHAN_SQL = <<~SQL.squish.freeze
+    parent_scope IS NOT NULL
+    AND BTRIM(parent_scope) <> ''
+    AND (scope IS NULL OR BTRIM(scope) = '')
+  SQL
+
+  def up
+    assert_no_orphan_parent_rows!
+
+    CONSTRAINTS.each do |table, name|
+      add_check_constraint table, INVARIANT_SQL, name: name, validate: false, if_not_exists: true
+
+      validate_check_constraint table, name: name
+    end
+  end
+
+  def down
+    CONSTRAINTS.each do |table, name|
+      remove_check_constraint table, name: name, if_exists: true
+    end
+  end
+
+  private
+
+  def assert_no_orphan_parent_rows!
+    violations = CONSTRAINTS.keys.filter_map do |table|
+      count = select_value("SELECT COUNT(*) FROM #{quote_table_name(table)} WHERE #{ORPHAN_SQL}").to_i
+      [table, count] if count.positive?
+    end
+    return if violations.empty?
+
+    evidence = violations.map { |table, count| "#{count} #{table} row(s) have parent_scope without scope" }.join("; ")
+    raise ActiveRecord::MigrationError,
+          "Cannot enforce the typed_eav parent-scope invariant: #{evidence}. Repair these rows and retry."
+  end
+end

data/db/migrate/20260816000000_use_partial_covering_scalar_indexes.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+class UsePartialCoveringScalarIndexes < ActiveRecord::Migration[7.1]
+  # Concurrent create-before-drop keeps the replacement indexes available
+  # throughout the upgrade and leaves rollback able to recreate legacy DDL.
+  disable_ddl_transaction!
+
+  TABLE = :typed_eav_values
+
+  INDEXES = [
+    {
+      column: :integer_value,
+      replacement: "idx_te_values_field_int_present",
+      legacy: "idx_te_values_field_int",
+      value_opclass: "int8_ops",
+    },
+    {
+      column: :decimal_value,
+      replacement: "idx_te_values_field_dec_present",
+      legacy: "idx_te_values_field_dec",
+      value_opclass: "numeric_ops",
+    },
+    {
+      column: :date_value,
+      replacement: "idx_te_values_field_date_present",
+      legacy: "idx_te_values_field_date",
+      value_opclass: "date_ops",
+    },
+    {
+      column: :datetime_value,
+      replacement: "idx_te_values_field_dt_present",
+      legacy: "idx_te_values_field_dt",
+      value_opclass: "timestamp_ops",
+    },
+    {
+      column: :boolean_value,
+      replacement: "idx_te_values_field_bool_present",
+      legacy: "idx_te_values_field_bool",
+      value_opclass: "bool_ops",
+    },
+    {
+      column: :string_value,
+      replacement: "idx_te_values_field_str_present",
+      legacy: "idx_te_values_field_str",
+      value_opclass: "text_pattern_ops",
+    },
+  ].freeze
+
+  def up
+    INDEXES.each { |definition| ensure_index!(replacement_definition(definition)) }
+    assert_all_valid!(INDEXES.map { |definition| replacement_definition(definition) })
+    INDEXES.each { |definition| remove_expected_index!(legacy_definition(definition)) }
+  end
+
+  def down
+    INDEXES.each { |definition| ensure_index!(legacy_definition(definition)) }
+    assert_all_valid!(INDEXES.map { |definition| legacy_definition(definition) })
+    INDEXES.each { |definition| remove_expected_index!(replacement_definition(definition)) }
+  end
+
+  private
+
+  def replacement_definition(definition)
+    definition.merge(
+      name: definition.fetch(:replacement),
+      include: ["entity_id"],
+      predicate: "#{definition.fetch(:column)} IS NOT NULL",
+    )
+  end
+
+  def legacy_definition(definition)
+    definition.merge(
+      name: definition.fetch(:legacy),
+      include: %w[entity_id entity_type],
+      predicate: nil,
+    )
+  end
+
+  def ensure_index!(expected)
+    actual = catalog_index(expected.fetch(:name))
+    return create_and_validate!(expected) unless actual
+    return if exact_definition?(actual, expected) && usable?(actual)
+
+    unless exact_definition?(actual, expected)
+      raise ActiveRecord::MigrationError,
+            "index #{expected.fetch(:name)} exists with unexpected definition: #{actual.inspect}"
+    end
+
+    say "repairing invalid interrupted index #{expected.fetch(:name)}"
+    remove_index TABLE, name: expected.fetch(:name), algorithm: :concurrently
+    create_and_validate!(expected)
+  end
+
+  def create_and_validate!(expected)
+    options = {
+      name: expected.fetch(:name),
+      include: expected.fetch(:include),
+      algorithm: :concurrently,
+    }
+    options[:where] = expected.fetch(:predicate) if expected.fetch(:predicate)
+    if expected.fetch(:value_opclass) == "text_pattern_ops"
+      options[:opclass] = { expected.fetch(:column) => :text_pattern_ops }
+    end
+
+    add_index TABLE, [:field_id, expected.fetch(:column)], **options
+    actual = catalog_index(expected.fetch(:name))
+    return if actual && exact_definition?(actual, expected) && usable?(actual)
+
+    raise ActiveRecord::MigrationError,
+          "concurrent index #{expected.fetch(:name)} was not created with its exact valid definition"
+  end
+
+  def assert_all_valid!(expected_indexes)
+    invalid = expected_indexes.filter_map do |expected|
+      actual = catalog_index(expected.fetch(:name))
+      expected.fetch(:name) unless actual && exact_definition?(actual, expected) && usable?(actual)
+    end
+    return if invalid.empty?
+
+    raise ActiveRecord::MigrationError,
+          "refusing to remove existing indexes before all replacements are valid: #{invalid.join(", ")}"
+  end
+
+  def remove_expected_index!(expected)
+    actual = catalog_index(expected.fetch(:name))
+    return unless actual
+
+    unless exact_definition?(actual, expected)
+      raise ActiveRecord::MigrationError,
+            "index #{expected.fetch(:name)} exists with unexpected definition: #{actual.inspect}"
+    end
+
+    remove_index TABLE, name: expected.fetch(:name), algorithm: :concurrently
+  end
+
+  def usable?(actual)
+    actual.fetch("valid") && actual.fetch("ready")
+  end
+
+  def exact_definition?(actual, expected)
+    actual.fetch("method") == "btree" &&
+      !actual.fetch("unique") &&
+      actual.fetch("columns") == ["field_id", expected.fetch(:column).to_s, *expected.fetch(:include)] &&
+      actual.fetch("key_count") == 2 &&
+      actual.fetch("opclasses") == ["int8_ops", expected.fetch(:value_opclass)] &&
+      normalized_predicate(actual.fetch("predicate")) == normalized_predicate(expected.fetch(:predicate))
+  end
+
+  def normalized_predicate(predicate)
+    predicate&.gsub(/[()\s"]/, "")&.downcase
+  end
+
+  def catalog_index(name)
+    quoted_name = connection.quote(name)
+    row = connection.select_one(<<~SQL.squish)
+      SELECT
+        index_state.indisvalid AS valid,
+        index_state.indisready AS ready,
+        index_state.indisunique AS unique,
+        access_method.amname AS method,
+        index_state.indnkeyatts AS key_count,
+        pg_get_expr(index_state.indpred, index_state.indrelid) AS predicate,
+        array_to_string(ARRAY(
+          SELECT attribute.attname
+          FROM unnest(index_state.indkey) WITH ORDINALITY AS index_column(attnum, position)
+          JOIN pg_attribute attribute
+            ON attribute.attrelid = index_state.indrelid
+           AND attribute.attnum = index_column.attnum
+          ORDER BY index_column.position
+        ), ',') AS columns,
+        array_to_string(ARRAY(
+          SELECT operator_class.opcname
+          FROM unnest(index_state.indclass) WITH ORDINALITY AS index_opclass(opcoid, position)
+          JOIN pg_opclass operator_class ON operator_class.oid = index_opclass.opcoid
+          WHERE index_opclass.position <= index_state.indnkeyatts
+          ORDER BY index_opclass.position
+        ), ',') AS opclasses
+      FROM pg_class index_relation
+      JOIN pg_index index_state ON index_state.indexrelid = index_relation.oid
+      JOIN pg_class table_relation ON table_relation.oid = index_state.indrelid
+      JOIN pg_namespace namespace ON namespace.oid = table_relation.relnamespace
+      JOIN pg_am access_method ON access_method.oid = index_relation.relam
+      WHERE namespace.nspname = ANY (current_schemas(false))
+        AND table_relation.relname = 'typed_eav_values'
+        AND index_relation.relname = #{quoted_name}
+    SQL
+    return unless row
+
+    row.merge(
+      "valid" => ActiveModel::Type::Boolean.new.cast(row.fetch("valid")),
+      "ready" => ActiveModel::Type::Boolean.new.cast(row.fetch("ready")),
+      "unique" => ActiveModel::Type::Boolean.new.cast(row.fetch("unique")),
+      "key_count" => row.fetch("key_count").to_i,
+      "columns" => row.fetch("columns").split(","),
+      "opclasses" => row.fetch("opclasses").split(","),
+    )
+  end
+end

data/lib/generators/typed_eav/scaffold/templates/controllers/typed_eav_controller.rb

@@ -133,9 +133,7 @@ class TypedEAVController < ApplicationController
   # `TypedEAV.config.scope_resolver` to set the tuple.
   def scoped_fields
     partition = current_partition!
-    return TypedEAV::Field::Base.all if partition[:mode] == :all_partitions
-
-    TypedEAV::Field::Base.where(id: visible_field_ids(partition))
+    TypedEAV::Partition.visible_fields(**partition)
   end
 
   # Resolve the ambient tuple for writes. Mirrors `scoped_fields` semantics:
@@ -183,12 +181,6 @@ class TypedEAVController < ApplicationController
     { scope: field.scope, parent_scope: field.parent_scope, mode: :partition }
   end
 
-  def visible_field_ids(partition)
-    TypedEAV::Field::Base.distinct.pluck(:entity_type).flat_map do |entity_type|
-      TypedEAV::Partition.visible_fields(entity_type: entity_type, **partition).pluck(:id)
-    end
-  end
-
   def resolve_type_class(type_name)
     return TypedEAV::Field::Text if type_name.blank?
 

data/lib/typed_eav/bulk_read.rb

@@ -6,13 +6,12 @@ module TypedEAV
   # `HasTypedEAV::InstanceMethods#typed_eav_hash`. N+1-free regardless of
   # record count or field count.
   #
-  # ## Pipeline (one query per unique partition tuple + one bulk value preload)
+  # ## Pipeline (one batched definition query + one bulk value preload)
   #
   #   1. validate_records!    — nil -> ArgumentError; single-class invariant
   #   2. group_by_tuple       — `[typed_eav_scope, typed_eav_parent_scope]`
-  #   3. winning_ids_by_tuple — `typed_eav_definitions` per unique tuple via
-  #                             `Partition.definitions_by_name` (shared
-  #                             collision-precedence helper)
+  #   3. winning_ids_by_tuple — one exact-partition definition query, then
+  #                             `Partition.definitions_by_name` per tuple
   #   4. preload_values       — single SELECT across ALL records
   #   5. build_result_hash    — per-record inner hash; orphan-skip + winning-id
   #                             precedence mirrored from the instance path.
@@ -21,9 +20,9 @@ module TypedEAV
   #
   #   - 1 SELECT typed_eav_values WHERE entity_type=? AND entity_id IN (?)
   #   - 1 SELECT typed_eav_fields WHERE id IN (?)           (via includes)
-  #   - 1 SELECT typed_eav_fields per unique partition tuple
+  #   - 1 SELECT typed_eav_fields for the union of requested partitions
   #
-  # Total: 2 + (unique partition tuples) queries — independent of record count.
+  # Total: 3 queries — independent of record count and partition cardinality.
   #
   # ## Single-class invariant
   #
@@ -74,10 +73,43 @@ module TypedEAV
     end
 
     def winning_ids_by_tuple(tuples)
-      tuples.each_with_object({}) do |(s, ps), memo|
-        defs = host_class.typed_eav_definitions(scope: s, parent_scope: ps)
-        memo[[s, ps]] = TypedEAV::Partition.definitions_by_name(defs).transform_values(&:id)
+      definitions_by_tuple = batched_definitions(tuples).group_by do |definition|
+        [definition.scope, definition.parent_scope]
       end
+
+      tuples.to_h do |tuple|
+        s, ps = tuple
+        candidate_tuples = [[nil, nil], [s, nil], [s, ps]].uniq
+        visible = candidate_tuples.flat_map { |candidate| definitions_by_tuple.fetch(candidate, []) }
+        [tuple, TypedEAV::Partition.definitions_by_name(visible).transform_values(&:id)]
+      end
+    end
+
+    def batched_definitions(tuples)
+      tuples.each { |scope, parent_scope| validate_tuple!(scope, parent_scope) }
+      relation = TypedEAV::Field::Base.where(entity_type: host_class.name)
+      requested = tuples.map { |scope, parent_scope| { "scope" => scope, "parent_scope" => parent_scope } }
+      relation.where(<<~SQL.squish, requested.to_json).to_a
+        typed_eav_fields.scope IS NULL AND typed_eav_fields.parent_scope IS NULL
+        OR EXISTS (
+          SELECT 1
+          FROM jsonb_to_recordset(?::jsonb) AS requested(scope text, parent_scope text)
+          WHERE (
+            typed_eav_fields.scope IS NOT DISTINCT FROM requested.scope
+            AND typed_eav_fields.parent_scope IS NOT DISTINCT FROM requested.parent_scope
+          )
+          OR (
+            typed_eav_fields.scope IS NOT DISTINCT FROM requested.scope
+            AND typed_eav_fields.parent_scope IS NULL
+          )
+        )
+      SQL
+    end
+
+    def validate_tuple!(scope, parent_scope)
+      return if TypedEAV::ScopeTuple.invariant_satisfied?(scope, parent_scope)
+
+      raise ArgumentError, "parent_scope cannot be set when scope is blank"
     end
 
     def preload_values(records)

data/lib/typed_eav/bulk_upsert.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module TypedEAV
+  # Reduced-semantics, SQL-oriented bulk value writer.
+  #
+  # This API runs casting/domain/entity/partition validation and validation
+  # callbacks, but does not run host callbacks/validations or host saves, Value
+  # persistence callbacks, versioning, or per-record error isolation. Callers must
+  # acknowledge those semantics explicitly. It pre-casts and validates every
+  # row in a transaction unit before issuing one upsert batch on the host
+  # model's connection. Scope resolution and field casting remain authoritative.
+  module BulkUpsert
+    TYPE_COLUMNS = %i[
+      string_value text_value boolean_value integer_value decimal_value date_value datetime_value json_value
+    ].freeze
+
+    class << self
+      # rubocop:disable Metrics/ParameterLists -- transaction controls are explicit API contract.
+      def execute(
+        host_class:, records:, values_by_field_name:, acknowledge_reduced_semantics:, transaction: :all, chunk_size: nil
+      )
+        validate_options!(records, values_by_field_name, acknowledge_reduced_semantics, transaction, chunk_size)
+        records = records.to_a
+        return 0 if records.empty?
+
+        validate_record_classes!(host_class, records)
+        reject_duplicate_or_unpersisted!(records)
+        ensure_connection_owner!(host_class)
+        values_by_field_name = normalize_values!(values_by_field_name)
+        units = transaction == :all ? [records] : records.each_slice(chunk_size)
+        units.sum { |unit| write_unit(host_class, unit, values_by_field_name) }
+      end
+
+      private
+
+      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- tuple grouping, validation, and row construction are one durability unit.
+      def write_unit(host_class, records, values_by_field_name)
+        tuples = records.map { |record| [record.typed_eav_scope, record.typed_eav_parent_scope] }.uniq
+        fields_by_tuple = tuples.to_h do |tuple|
+          # Use the exact tuple at the partition layer. EntityQuery treats
+          # explicit scope kwargs as ALL_SCOPES inside TypedEAV.unscoped;
+          # that administrative bypass must not cross-contaminate tenants.
+          fields = TypedEAV::Partition.definitions_by_name(
+            TypedEAV::Partition.visible_fields(
+              entity_type: host_class.polymorphic_name,
+              scope: tuple[0],
+              parent_scope: tuple[1],
+            ),
+          )
+          [tuple, fields]
+        end
+        rows = records.flat_map do |record|
+          fields = fields_by_tuple.fetch([record.typed_eav_scope, record.typed_eav_parent_scope])
+          values_by_field_name.filter_map do |name, raw|
+            field = fields[name.to_s]
+            raise ArgumentError, "bulk_upsert field is not visible for #{record.class.name}: #{name}" unless field
+
+            build_row(host_class, record, field, raw)
+          end
+        end
+        return 0 if rows.empty?
+
+        host_class.transaction do
+          # rubocop:disable Rails/SkipsModelValidations -- every row is prevalidated in :bulk_upsert context.
+          TypedEAV::Value.upsert_all(
+            rows,
+            unique_by: :idx_te_values_entity_field,
+            update_only: (TYPE_COLUMNS + [:updated_at]),
+            record_timestamps: false,
+          )
+          # rubocop:enable Rails/SkipsModelValidations
+        end
+        rows.length
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+      def build_row(host_class, record, field, raw)
+        value = TypedEAV::Value.new(entity: record, field: field)
+        value.value = raw
+        value.valid?(:bulk_upsert)
+        errors = value.errors.to_hash
+        raise ArgumentError, "bulk_upsert validation failed for #{field.name}: #{errors.inspect}" if errors.any?
+
+        now = Time.current
+        row = {
+          entity_type: host_class.polymorphic_name,
+          entity_id: record.id,
+          field_id: field.id,
+          created_at: now,
+          updated_at: now,
+        }
+        TYPE_COLUMNS.each { |column| row[column] = value[column] }
+        row
+      end
+
+      def validate_options!(records, values, acknowledged, transaction, chunk_size)
+        raise ArgumentError, "bulk_upsert requires an Enumerable of records" if records.nil?
+        raise ArgumentError, "bulk_upsert requires a Hash of values_by_field_name" unless values.is_a?(Hash)
+        raise ArgumentError, "bulk_upsert requires acknowledge_reduced_semantics: true" unless acknowledged == true
+        return if transaction == :all
+        return if transaction == :chunks && chunk_size.is_a?(Integer) && chunk_size.positive?
+
+        raise ArgumentError, "transaction must be :all or :chunks with a positive chunk_size"
+      end
+
+      def normalize_values!(values)
+        normalized = {}
+        values.each do |name, raw|
+          key = name.to_s
+          if normalized.key?(key)
+            raise ArgumentError, "bulk_upsert field keys collide after string normalization: #{key}"
+          end
+
+          normalized[key] = raw
+        end
+        normalized
+      end
+
+      def validate_record_classes!(host_class, records)
+        return if records.all?(host_class)
+
+        raise ArgumentError, "bulk_upsert expects records of class #{host_class.name}"
+      end
+
+      def reject_duplicate_or_unpersisted!(records)
+        raise ArgumentError, "bulk_upsert requires persisted records" if records.any? { |record| !record.persisted? }
+        return if records.map(&:id).uniq.length == records.length
+
+        raise ArgumentError, "bulk_upsert does not accept duplicate entity records"
+      end
+
+      def ensure_connection_owner!(host_class)
+        pools = [host_class.connection_pool, TypedEAV::Value.connection_pool, TypedEAV::Field::Base.connection_pool]
+        return if pools.map(&:object_id).uniq.one?
+
+        raise ArgumentError, "bulk_upsert requires host, value, and field connections to share a pool"
+      end
+    end
+    # rubocop:enable Metrics/ParameterLists
+  end
+end