hecks 3.5.0 → 3.6.0

data/lib/hecks/ports/persistence/plugins/era/postgres_era/lineage/provisioning.rb

@@ -1,3 +1,6 @@
+require_relative "base_sql"
+require_relative "domain_rename"
+
 module Hecks
   module Adapters
     class PostgresEra
@@ -5,6 +8,9 @@ module Hecks
         # Owner-only schema bootstrap: lineage tables, the RLS fence, the domain rename
         # bridge, and per-era journal partitions.
         module Provisioning
+          include BaseSql
+          include DomainRename
+
           # Builds or tops up this domain's lineage schema when the connected role owns the
           # journal (or none exists yet); any other role only attempts the rename bridge.
           #
@@ -14,169 +20,18 @@ module Hecks
           # @raise [Runtime::WiringError] if the `formerly_known_as` rename cannot take
           #   its locks within 10s or Postgres refuses one of its statements
           # @raise [PG::Error] if Postgres refuses a provisioning statement
-          # rubocop:disable-next Metrics/MethodLength
           def ensure_base!
             rename_domain! if @formerly_known_as
             return unless provisioner?
 
-            @db.exec(<<~SQL)
-              CREATE TABLE IF NOT EXISTS hecks_eras (
-                domain    text NOT NULL,
-                ordinal   int  NOT NULL,
-                hash      text,
-                label     text,
-                held_text text NOT NULL,
-                watermark bigint,
-                PRIMARY KEY (domain, ordinal)
-              )
-            SQL
-            @db.exec("ALTER TABLE hecks_eras ADD COLUMN IF NOT EXISTS held_digest text")
-            @db.exec("ALTER TABLE hecks_eras ADD COLUMN IF NOT EXISTS held_projection jsonb")
-            # Canonical-form version that minted the name (Runtime::StorageShape::FORM_VERSION);
-            # NULL reads as 1.
-            @db.exec("ALTER TABLE hecks_eras ADD COLUMN IF NOT EXISTS canon_form int")
-            # Every frozen text version, archived where an edit cannot reach it.
-            @db.exec(<<~SQL)
-              CREATE TABLE IF NOT EXISTS hecks_era_texts (
-                domain      text NOT NULL,
-                ordinal     int  NOT NULL,
-                digest      text NOT NULL,
-                held_text   text NOT NULL,
-                archived_at timestamptz NOT NULL DEFAULT now(),
-                PRIMARY KEY (domain, ordinal, digest)
-              )
-            SQL
-            @db.exec(<<~SQL)
-              CREATE TABLE IF NOT EXISTS hecks_approvals (
-                domain           text NOT NULL,
-                from_label       text NOT NULL,
-                to_label         text NOT NULL,
-                edge_digest      text NOT NULL,
-                reviewed_ordinal bigint NOT NULL,
-                approved_at      timestamptz NOT NULL DEFAULT now()
-              )
-            SQL
-            @db.exec("CREATE SEQUENCE IF NOT EXISTS #{quote(sequence)}")
-            # An owned sequence default, not generated always as identity, which
-            # partitioned tables only support from Postgres 17.
-            @db.exec(<<~SQL)
-              CREATE TABLE IF NOT EXISTS #{quoted_journal} (
-                ordinal      bigint NOT NULL DEFAULT nextval('#{sequence}'),
-                era          int    NOT NULL,
-                aggregate    text   NOT NULL,
-                aggregate_id text   NOT NULL,
-                operation    text   NOT NULL DEFAULT 'save',
-                state        jsonb,
-                mirrors      jsonb
-              ) PARTITION BY LIST (era)
-            SQL
+            create_lineage_tables!
+            create_journal!
             ensure_partition!(1)
-            # Journal rows are immutable by privilege: nothing updates or deletes them.
-            # The REVOKE is guarded because it rewrites pg_class.relacl and takes a lock
-            # even when nothing changes, so concurrent reboots would race into
-            # `tuple concurrently updated`.
-            #
-            # The guard is `relacl IS NULL`, not has_table_privilege('public', ..., 'UPDATE'):
-            # a new table already answers "no UPDATE", so that check would skip the first
-            # REVOKE and leave relacl NULL forever. pg_table_is_visible(oid), not a bare
-            # relname match, so a sibling schema's same-named table is never mistaken for ours.
-            relacl_null = @db.exec_params(
-              "SELECT relacl IS NULL FROM pg_class WHERE relname = $1 AND pg_table_is_visible(oid)", [journal]
-            ).getvalue(0, 0)
-            @db.exec("REVOKE UPDATE, DELETE ON #{quoted_journal} FROM PUBLIC") if relacl_null == "t"
-            # RLS goes on at provisioning, never mid-life: enabling it later would deny
-            # every role that has no policy yet.
-            #
-            # FORCE, not just enable, or the table owner is exempt from every policy. A
-            # superuser or BYPASSRLS role stays exempt regardless (see check_fence_applies!).
-            #
-            # Guarded: enable/FORCE takes an AccessExclusiveLock even as a no-op, and this
-            # runs on every boot, so an unconditional reissue would freeze concurrent writers.
-            current = @db.exec_params(
-              "SELECT relrowsecurity, relforcerowsecurity FROM pg_class " \
-              "WHERE relname = $1 AND pg_table_is_visible(oid)", [journal]
-            )[0]
-            @db.exec("ALTER TABLE #{quoted_journal} ENABLE ROW LEVEL SECURITY") unless current["relrowsecurity"] == "t"
-            @db.exec("ALTER TABLE #{quoted_journal} FORCE ROW LEVEL SECURITY") unless current["relforcerowsecurity"] == "t"
+            revoke_journal_mutation!
+            force_row_security!
             install_transforms!
           end
 
-          # Moves a domain's journal, sequence, partitions and lineage rows from
-          # `formerly_known_as` to the current name, in one transaction.
-          #
-          # Runs before `provisioner?` and the CREATE TABLE calls in `ensure_base!`; a no-op when
-          # `hecks_eras` is missing, the new name has rows, or the former name has none.
-          # Locks go in a fixed order (old before new, eras before ordinal) so nothing deadlocks.
-          #
-          # @return [void]
-          # @raise [Runtime::WiringError] if the locks are not granted within 10s or Postgres
-          #   refuses a statement (prechecks included); both roll back first
-          # rubocop:disable-next Metrics/AbcSize
-          # rubocop:disable-next Metrics/MethodLength
-          def rename_domain!
-            return unless @db.exec_params("SELECT to_regclass($1)", ["hecks_eras"])[0]["to_regclass"]
-            return if @db.exec_params(
-              "SELECT 1 FROM hecks_eras WHERE domain = $1 LIMIT 1", [@domain]
-            ).ntuples.positive?
-            return if @db.exec_params(
-              "SELECT 1 FROM hecks_eras WHERE domain = $1 LIMIT 1", [@formerly_known_as]
-            ).ntuples.zero?
-
-            old_journal  = "hecks_journal_#{Naming.snake(@formerly_known_as)}"
-            old_sequence = "#{old_journal}_ordinal"
-            ordinals = @db.exec_params(
-              "SELECT ordinal FROM hecks_eras WHERE domain = $1 ORDER BY ordinal", [@formerly_known_as]
-            ).map { |row| row["ordinal"].to_i }
-
-            @db.exec("BEGIN")
-            @db.exec("SET LOCAL lock_timeout = '10s'")
-            [
-              "hecks_eras:#{@formerly_known_as}", "hecks_eras:#{@domain}",
-              "hecks_ordinal:#{@formerly_known_as}", "hecks_ordinal:#{@domain}"
-            ].each do |key|
-              @db.exec_params("SELECT pg_advisory_xact_lock(hashtext($1))", [key])
-            end
-
-            @db.exec("ALTER TABLE #{quote(old_journal)} RENAME TO #{quote(journal)}")
-            # The sequence is a plain create sequence, not owned by the column, so it
-            # does not move with the table and needs its own rename.
-            @db.exec("ALTER SEQUENCE #{quote(old_sequence)} RENAME TO #{quote(sequence)}")
-            # Renaming the parent does not rename its partitions.
-            ordinals.each do |ordinal|
-              old_partition = "#{old_journal}_era_#{ordinal}"
-              @db.exec("ALTER TABLE #{quote(old_partition)} RENAME TO #{quote(partition(ordinal))}")
-            end
-
-            %w[hecks_eras hecks_era_texts hecks_approvals].each do |table|
-              @db.exec_params("UPDATE #{table} SET domain = $1 WHERE domain = $2", [@domain, @formerly_known_as])
-            end
-            # hecks_attestations is created lazily on first reattest!, so it may not exist.
-            if @db.exec_params(
-              "SELECT to_regclass($1)", ["hecks_attestations"]
-            )[0]["to_regclass"]
-              @db.exec_params("UPDATE hecks_attestations SET domain = $1 WHERE domain = $2",
-                              [@domain, @formerly_known_as])
-            end
-
-            @db.exec("COMMIT")
-          rescue PG::LockNotAvailable
-            begin
-              @db.exec("ROLLBACK")
-            rescue StandardError
-              nil
-            end
-            raise Runtime::WiringError,
-                  "cannot rename #{@formerly_known_as} to #{@domain}: another rename, mint, or write holds " \
-                  "one of the domain locks — waited 10s; try again shortly"
-          rescue PG::Error => e
-            begin
-              @db.exec("ROLLBACK")
-            rescue StandardError
-              nil
-            end
-            raise Runtime::WiringError, "cannot rename #{@formerly_known_as} to #{@domain}: #{e.message.strip}"
-          end
-
           # Refuses to boot on a connection whose role Postgres exempts from row-level security
           # (a superuser or BYPASSRLS role) unless the caller opts in.
           #
@@ -191,26 +46,12 @@ module Hecks
           # @raise [PG::Error] if the `pg_roles` lookup fails
           def check_fence_applies!(allow_superuser: false)
             row = @db.exec("SELECT rolname, rolsuper, rolbypassrls FROM pg_roles WHERE rolname = current_user")[0]
-            exempt = []
-            exempt << "a superuser" if row["rolsuper"] == "t"
-            exempt << "granted BYPASSRLS" if row["rolbypassrls"] == "t"
+            exempt = fence_exemptions(row)
             return if exempt.empty?
 
             role = row["rolname"].inspect
-            unless allow_superuser
-              raise Runtime::WiringError,
-                    "cannot boot #{@domain}: PostgresEra's era write-fence is row-level security, and this " \
-                    "connection's role #{role} is #{exempt.join(' and ')} — Postgres exempts it from every " \
-                    "policy, FORCE included, so an old checkout connected this way keeps writing a superseded " \
-                    "era and nothing refuses. Connect as an ordinary role instead (database " \
-                    "\"postgres://<role>@<host>/<db>\" in the .world — a non-superuser OWNER still provisions " \
-                    "and mints), or declare `allow_superuser true` in the same persisted_by block to boot with " \
-                    "the fence void, on the record."
-            end
-
-            warn "[hecks] #{@domain}: booting PostgresEra as #{role}, #{exempt.join(' and ')}, under " \
-                 "allow_superuser — the era write-fence is void for this connection; only this process's own " \
-                 "superseded-era check (PostgresEra#append) stands between an old checkout and a superseded era"
+            refuse_unfenced_role!(role, exempt) unless allow_superuser
+            warn_unfenced_role(role, exempt)
           end
 
           # Tells whether this connection's role is the one that builds the schema:
@@ -242,15 +83,8 @@ module Hecks
           def ensure_partition!(era)
             return if partition_attached?(era)
 
-            @db.exec(<<~SQL)
-              CREATE TABLE IF NOT EXISTS #{quote(partition(era))} (
-                LIKE #{quoted_journal} INCLUDING DEFAULTS
-              )
-            SQL
-            @db.exec(<<~SQL)
-              ALTER TABLE #{quoted_journal}
-                ATTACH PARTITION #{quote(partition(era))} FOR VALUES IN (#{era.to_i})
-            SQL
+            @db.exec(format(PARTITION_SQL, partition: quote(partition(era)), journal: quoted_journal))
+            @db.exec(format(ATTACH_PARTITION_SQL, journal: quoted_journal, partition: quote(partition(era)), era: era.to_i))
           end
 
           # Tells whether an era's partition is part of the journal, by asking
@@ -273,6 +107,88 @@ module Hecks
               [partition(era), journal]
             ).ntuples.positive?
           end
+
+          private
+
+          def create_lineage_tables!
+            @db.exec(ERAS_SQL)
+            @db.exec("ALTER TABLE hecks_eras ADD COLUMN IF NOT EXISTS held_digest text")
+            @db.exec("ALTER TABLE hecks_eras ADD COLUMN IF NOT EXISTS held_projection jsonb")
+            # Canonical-form version that minted the name (Runtime::StorageShape::FORM_VERSION);
+            # NULL reads as 1.
+            @db.exec("ALTER TABLE hecks_eras ADD COLUMN IF NOT EXISTS canon_form int")
+            @db.exec(ERA_TEXTS_SQL)
+            @db.exec(APPROVALS_SQL)
+          end
+
+          def create_journal!
+            @db.exec("CREATE SEQUENCE IF NOT EXISTS #{quote(sequence)}")
+            @db.exec(format(JOURNAL_SQL, journal: quoted_journal, sequence: sequence))
+          end
+
+          # Journal rows are immutable by privilege: nothing updates or deletes them.
+          # The REVOKE is guarded because it rewrites pg_class.relacl and takes a lock
+          # even when nothing changes, so concurrent reboots would race into
+          # `tuple concurrently updated`.
+          #
+          # The guard is `relacl IS NULL`, not has_table_privilege('public', ..., 'UPDATE'):
+          # a new table already answers "no UPDATE", so that check would skip the first
+          # REVOKE and leave relacl NULL forever. pg_table_is_visible(oid), not a bare
+          # relname match, so a sibling schema's same-named table is never mistaken for ours.
+          def revoke_journal_mutation!
+            relacl_null = @db.exec_params(
+              "SELECT relacl IS NULL FROM pg_class WHERE relname = $1 AND pg_table_is_visible(oid)", [journal]
+            ).getvalue(0, 0)
+            @db.exec("REVOKE UPDATE, DELETE ON #{quoted_journal} FROM PUBLIC") if relacl_null == "t"
+          end
+
+          # RLS goes on at provisioning, never mid-life: enabling it later would deny
+          # every role that has no policy yet.
+          #
+          # FORCE, not just enable, or the table owner is exempt from every policy. A
+          # superuser or BYPASSRLS role stays exempt regardless (see check_fence_applies!).
+          #
+          # Guarded: enable/FORCE takes an AccessExclusiveLock even as a no-op, and this
+          # runs on every boot, so an unconditional reissue would freeze concurrent writers.
+          def force_row_security!
+            current = @db.exec_params(
+              "SELECT relrowsecurity, relforcerowsecurity FROM pg_class " \
+              "WHERE relname = $1 AND pg_table_is_visible(oid)", [journal]
+            )[0]
+            @db.exec("ALTER TABLE #{quoted_journal} ENABLE ROW LEVEL SECURITY") unless current["relrowsecurity"] == "t"
+            @db.exec("ALTER TABLE #{quoted_journal} FORCE ROW LEVEL SECURITY") unless current["relforcerowsecurity"] == "t"
+          end
+
+          def fence_exemptions(row)
+            exempt = []
+            exempt << "a superuser" if row["rolsuper"] == "t"
+            exempt << "granted BYPASSRLS" if row["rolbypassrls"] == "t"
+            exempt
+          end
+
+          def refuse_unfenced_role!(role, exempt)
+            raise Runtime::WiringError,
+                  "cannot boot #{@domain}: PostgresEra's era write-fence is row-level security, and this " \
+                  "connection's role #{role} is #{exempt.join(" and ")} — Postgres exempts it from every " \
+                  "policy, FORCE included, so an old checkout connected this way keeps writing a superseded " \
+                  "era and nothing refuses. Connect as an ordinary role instead (database " \
+                  "\"postgres://<role>@<host>/<db>\" in the .world — a non-superuser OWNER still provisions " \
+                  "and mints), or declare `allow_superuser true` in the same persisted_by block to boot with " \
+                  "the fence void, on the record."
+          end
+
+          def warn_unfenced_role(role, exempt)
+            warn "[hecks] #{@domain}: booting PostgresEra as #{role}, #{exempt.join(" and ")}, under " \
+                 "allow_superuser — the era write-fence is void for this connection; only this process's own " \
+                 "superseded-era check (PostgresEra#append) stands between an old checkout and a superseded era"
+          end
+
+          # Abandons the open transaction, ignoring a failure of the rollback itself.
+          def rollback_quietly
+            @db.exec("ROLLBACK")
+          rescue StandardError
+            nil
+          end
         end
       end
     end

data/lib/hecks/ports/persistence/plugins/era/postgres_era/lineage/resumable_backfill.rb

@@ -39,36 +39,36 @@ module Hecks
           # One chunk, one transaction. Progress is re-read under the lock: a concurrent booter may
           # have committed this chunk while we waited.
           def run_chunk!(target, source_sql:, upsert:)
-            progress = backfill_progress(target)
-            return true if progress[:completed]
+            return true if backfill_progress(target)[:completed]
 
             completed = false
             nested_transaction("hecks_backfill_chunk") do
               # Prefix is disjoint from the ordinal/era/head-snapshot lock keys, so a chunk
               # never blocks a write or a mint.
               @db.exec_params("SELECT pg_advisory_xact_lock(hashtext('hecks_field_cache:' || $1))", [target])
-              progress = backfill_progress(target)
-              if progress[:completed]
-                completed = true
-                next
-              end
+              completed = fill_chunk!(target, source_sql, upsert)
+            end
+            completed
+          end
 
-              rows = @db.exec(source_sql.call(progress[:cursor]))
-              if rows.ntuples.zero?
-                upsert_backfill_progress!(target, cursor: progress[:cursor], completed: true)
-                completed = true
-                next
-              end
+          # Reads the next chunk under the lock, upserts it and advances the cursor; answers
+          # whether the backfill is now complete.
+          def fill_chunk!(target, source_sql, upsert)
+            progress = backfill_progress(target)
+            return true if progress[:completed]
 
-              upsert.call(rows)
-              completed = rows.ntuples < CHUNK_SIZE
-              # `PG::Result#[]` rejects negative indexes, so index the last row explicitly.
-              last_cursor = rows[rows.ntuples - 1]["id"]
-              upsert_backfill_progress!(target, cursor: last_cursor, completed: completed)
-            end
+            rows = @db.exec(source_sql.call(progress[:cursor]))
+            upsert.call(rows) unless rows.ntuples.zero?
+            completed = rows.ntuples < CHUNK_SIZE
+            upsert_backfill_progress!(target, cursor: last_id(rows, progress[:cursor]), completed: completed)
             completed
           end
 
+          # `PG::Result#[]` rejects negative indexes, so index the last row explicitly.
+          def last_id(rows, fallback)
+            rows.ntuples.zero? ? fallback : rows[rows.ntuples - 1]["id"]
+          end
+
           # `PG::Result#[]` raises IndexError out of range, so check `ntuples.zero?` before `[0]`.
           def backfill_progress(target)
             result = @db.exec_params(

data/lib/hecks/ports/persistence/plugins/era/postgres_era/lineage/sql_templates.rb

@@ -0,0 +1,88 @@
+module Hecks
+  module Adapters
+    class PostgresEra
+      class Lineage
+        # The SQL templates of the head and its chain, filled with `format`.
+        module SqlTemplates
+          # The DDL for one era's head-snapshot table. A delete upserts a tombstone
+          # (operation = 'delete', state NULL) instead of removing the row, so it still
+          # outranks a stale ancestor `save` row by ordinal under DISTINCT ON.
+          HEAD_SNAPSHOT_SQL = <<~SQL.freeze
+            CREATE TABLE %<table>s (
+              id        text PRIMARY KEY,
+              ordinal   bigint NOT NULL,
+              operation text NOT NULL DEFAULT 'save',
+              state     jsonb
+            )
+          SQL
+
+          # One chunk of the journal's latest save per id, after `cursor` when the backfill resumes.
+          HEAD_BACKFILL_SQL = <<~SQL.freeze
+            SELECT id, ordinal, state FROM (
+              SELECT DISTINCT ON (aggregate_id) aggregate_id AS id, ordinal, operation, state
+              FROM %<journal>s
+              WHERE era = %<era>s AND aggregate = %<aggregate>s
+              %<after_cursor>s
+              ORDER BY aggregate_id, ordinal DESC
+            ) latest WHERE operation = 'save' ORDER BY id LIMIT %<chunk_size>s
+          SQL
+
+          # Era 1's head view: the snapshot table itself. WHERE operation = 'save' excludes
+          # delete tombstones, whose state is NULL, so a deleted id falls out of the head
+          # instead of resolving to a nil state.
+          FIRST_HEAD_VIEW_SQL = <<~SQL.freeze
+            CREATE OR REPLACE VIEW %<head_view>s AS
+            SELECT id, state FROM %<snapshot>s WHERE operation = 'save'
+          SQL
+
+          # The head view of era N: the era's matview merged with its own snapshot. Reads the
+          # snapshot's own operation column rather than hardcoding 'save', so a tombstone here
+          # outranks a stale ancestor save row by ordinal instead of letting it resurrect a
+          # deleted record.
+          HEAD_VIEW_SQL = <<~SQL.freeze
+            CREATE VIEW %<head_view>s AS
+            SELECT id, state FROM (
+              SELECT DISTINCT ON (aggregate_id) aggregate_id AS id, operation, state FROM (
+                SELECT ordinal, aggregate_id, operation, state FROM %<view>s
+                UNION ALL
+                SELECT ordinal, id AS aggregate_id, operation, state
+                FROM %<snapshot>s
+              ) merged ORDER BY aggregate_id, ordinal DESC
+            ) latest WHERE operation = 'save'
+          SQL
+
+          # Era N layered onto era N-1's existing matview.
+          LAYERED_SQL = <<~SQL.freeze
+            WITH layered AS (
+              SELECT DISTINCT ON (aggregate_id) ordinal, aggregate_id, operation, state FROM (
+                SELECT ordinal, aggregate_id, operation, state FROM %<prior_view>s
+                UNION ALL
+                SELECT ordinal, aggregate_id, operation, state FROM %<journal>s
+                WHERE era = %<prior_era>s AND aggregate = %<aggregate>s%<cut>s
+              ) layers ORDER BY aggregate_id, ordinal DESC
+            )
+            SELECT ordinal, %<id_column>s, operation,
+                   CASE WHEN operation = 'save' THEN %<expression>s ELSE state END AS state
+            FROM layered
+          SQL
+
+          # Every edge chained over the reduced ancestor tail.
+          CHAIN_SQL = <<~SQL.freeze
+            WITH tail AS (%<tail>s),
+            %<chain>s
+            SELECT ordinal, aggregate_id, operation, state FROM edge_%<last>s
+          SQL
+
+          # Reduces to newest entry per aggregate id, dropping any id whose newest entry is a
+          # delete.
+          LATEST_SQL = <<~SQL.freeze
+            SELECT aggregate_id, state FROM (
+              SELECT DISTINCT ON (aggregate_id) aggregate_id, operation, state
+              FROM (%<sql>s) chained ORDER BY aggregate_id, ordinal DESC
+            ) latest WHERE operation = 'save'
+          SQL
+        end
+      end
+    end
+  end
+end