hecks 0.2.0 → 1.0.0

data/lib/hecks/ports/persistence/plugins/era/postgres_era/lineage/provisioning.rb

@@ -0,0 +1,314 @@
+module Hecks
+  module Adapters
+    class PostgresEra
+      class Lineage
+        module Provisioning
+          # Provisioning is the OWNER's job, and a deployment's app role is
+          # deliberately not the owner — it may append and read, and it
+          # owns nothing. By the time such a role connects, the base is
+          # already built, so its boot verifies rather than builds.
+          #
+          # Without this guard the per-era fence below is unreachable: the
+          # ALTER TABLE and REVOKE here are owner-only, so the very role
+          # grant_era! exists to constrain could never finish booting
+          # ("must be owner of table hecks_eras"). A fence nothing can
+          # reach is not a fence.
+          def ensure_base!
+            rename_domain! if @formerly_known_as
+            return unless provisioner?
+
+            @db.exec(<<~SQL)
+              CREATE TABLE IF NOT EXISTS hecks_eras (
+                domain    text NOT NULL,
+                ordinal   int  NOT NULL,
+                hash      text,
+                label     text,
+                held_text text NOT NULL,
+                watermark bigint,
+                PRIMARY KEY (domain, ordinal)
+              )
+            SQL
+            @db.exec("ALTER TABLE hecks_eras ADD COLUMN IF NOT EXISTS held_digest text")
+            @db.exec("ALTER TABLE hecks_eras ADD COLUMN IF NOT EXISTS held_projection jsonb")
+            # which canonical-form version minted this name — see
+            # Runtime::StorageShape::FORM_VERSION; rows minted before the
+            # column carry NULL, read as an implicit 1
+            @db.exec("ALTER TABLE hecks_eras ADD COLUMN IF NOT EXISTS canon_form int")
+            # every frozen text version, archived where an edit cannot
+            # reach it — the recovery the hard reattest refusal points at
+            @db.exec(<<~SQL)
+              CREATE TABLE IF NOT EXISTS hecks_era_texts (
+                domain      text NOT NULL,
+                ordinal     int  NOT NULL,
+                digest      text NOT NULL,
+                held_text   text NOT NULL,
+                archived_at timestamptz NOT NULL DEFAULT now(),
+                PRIMARY KEY (domain, ordinal, digest)
+              )
+            SQL
+            @db.exec(<<~SQL)
+              CREATE TABLE IF NOT EXISTS hecks_approvals (
+                domain           text NOT NULL,
+                from_label       text NOT NULL,
+                to_label         text NOT NULL,
+                edge_digest      text NOT NULL,
+                reviewed_ordinal bigint NOT NULL,
+                approved_at      timestamptz NOT NULL DEFAULT now()
+              )
+            SQL
+            @db.exec("CREATE SEQUENCE IF NOT EXISTS #{quote(sequence)}")
+            # GENERATED ALWAYS AS IDENTITY is the intent, but identity
+            # columns on partitioned tables need Postgres 17 — an owned
+            # sequence default is the same spanning ordinal on any
+            # supported server.
+            @db.exec(<<~SQL)
+              CREATE TABLE IF NOT EXISTS #{quoted_journal} (
+                ordinal      bigint NOT NULL DEFAULT nextval('#{sequence}'),
+                era          int    NOT NULL,
+                aggregate    text   NOT NULL,
+                aggregate_id text   NOT NULL,
+                operation    text   NOT NULL DEFAULT 'save',
+                state        jsonb,
+                mirrors      jsonb
+              ) PARTITION BY LIST (era)
+            SQL
+            ensure_partition!(1)
+            # Immutability by privilege: nothing updates or deletes journal
+            # rows. The owner's implicit rights remain (Postgres has no way
+            # to revoke them from the owner itself); a deployment's app
+            # role connects as a NON-owner and gets exactly INSERT, per
+            # era, at mint time.
+            #
+            # GUARDED, same reasoning as the RLS ALTER TABLE calls just
+            # below: REVOKE still writes pg_class.relacl (and takes the
+            # matching lock) even when the resulting privileges are
+            # unchanged, so an unconditional reissue on every ordinary
+            # reboot raced two concurrent boots into
+            # `PG::InternalError: tuple concurrently updated` — read
+            # first, touch the catalog only on the boot that actually
+            # needs to.
+            #
+            # `relacl IS NULL`, not `has_table_privilege('public', ...,
+            # 'UPDATE')` — found live, not assumed: a brand-new table's
+            # PUBLIC privilege is already "no UPDATE" by Postgres's own
+            # default (nothing has ever been explicitly granted to
+            # PUBLIC), so `has_table_privilege` answers false BOTH before
+            # the REVOKE has ever run AND after it has — indistinguishable
+            # by that check alone, which meant the very first boot's own
+            # REVOKE never actually ran, `relacl` stayed NULL forever, and
+            # "journal rows accept no UPDATE/DELETE from PUBLIC" was true
+            # only by accident of Postgres's default, not by the explicit
+            # privilege revocation this method exists to record.
+            # `relacl IS NULL` means "default ACL, nothing explicit yet" —
+            # exactly the one-time signal needed, and (like the RLS flags
+            # below) `pg_table_is_visible(oid)`, not a bare relname match,
+            # for the same shared-instance/storehouse reason.
+            relacl_null = @db.exec_params(
+              "SELECT relacl IS NULL FROM pg_class WHERE relname = $1 AND pg_table_is_visible(oid)", [journal]
+            ).getvalue(0, 0)
+            @db.exec("REVOKE UPDATE, DELETE ON #{quoted_journal} FROM PUBLIC") if relacl_null == "t"
+            # RLS goes on AT PROVISIONING, never mid-life — enabling it
+            # later would deny every role that has no policy yet, on
+            # whatever the shape of the schema happened to be at that
+            # moment.
+            #
+            # FORCE, not merely ENABLE: without it, the table OWNER is
+            # exempt from every policy here, by Postgres default — which
+            # would leave the schema writable forever to whoever holds
+            # the owner's credentials, the one connection this whole
+            # design cannot fence. Checked, not assumed: mint_era! never
+            # inserts into the journal at all (only hecks_eras/
+            # hecks_era_texts, neither RLS-protected), and merge_tail!'s
+            # one journal INSERT targets the CURRENT era, which the fence
+            # already admits for anyone with base privileges — so FORCE
+            # costs the owner nothing operations here actually need.
+            #
+            # This still exempts an actual Postgres SUPERUSER (or any
+            # role granted BYPASSRLS) unconditionally — FORCE only
+            # narrows what ENABLE already narrows for the owner
+            # specifically, and superuser bypass sits above both. Running
+            # migrations as a real superuser (self-hosted Postgres, most
+            # commonly) leaves this gap open regardless; a managed
+            # provider's admin account is typically NOT a superuser, and
+            # is exactly what FORCE closes.
+            #
+            # GUARDED, not reissued unconditionally — measured, not
+            # assumed: `ALTER TABLE ... ENABLE/FORCE ROW LEVEL SECURITY`
+            # takes AccessExclusiveLock EVEN WHEN THE SETTING IS ALREADY
+            # CORRECT (Postgres does not skip the lock just because the
+            # statement would be a no-op). ensure_base! runs on EVERY
+            # boot by the owning role, not only the first — so an
+            # unconditional reissue here would mean every ordinary
+            # reboot of the deployment's own identity re-freezes every
+            # concurrent writer, on any era, for as long as that ALTER
+            # TABLE has to wait its turn. Read the current state first;
+            # touch the catalog only on the boot that actually needs to.
+            # pg_table_is_visible, NOT a bare relname match — a shared
+            # instance (storehouse) can hold a same-named journal table
+            # per schema; catalog lookups here must resolve the SAME way
+            # search_path resolves an unqualified SQL reference, or a
+            # sibling domain's table satisfies a query meant for this
+            # domain's own.
+            current = @db.exec_params(
+              "SELECT relrowsecurity, relforcerowsecurity FROM pg_class " \
+              "WHERE relname = $1 AND pg_table_is_visible(oid)", [journal]
+            )[0]
+            @db.exec("ALTER TABLE #{quoted_journal} ENABLE ROW LEVEL SECURITY") unless current["relrowsecurity"] == "t"
+            @db.exec("ALTER TABLE #{quoted_journal} FORCE ROW LEVEL SECURITY") unless current["relforcerowsecurity"] == "t"
+            install_transforms!
+          end
+
+          # A DOMAIN'S OWN IDENTITY CHANGED — bridge its history under the
+          # new name, before `provisioner?`/the CREATE TABLE IF NOT EXISTS
+          # block below ever run. That ordering is load-bearing, not
+          # tidiness: `provisioner?` and every statement in ensure_base!
+          # test for the journal under `journal` — the NEW name — and a
+          # freshly-renamed domain looks, to those checks, exactly like a
+          # domain that has never been provisioned at all. Left where it
+          # was written, ensure_base! would happily CREATE TABLE IF NOT
+          # EXISTS a brand-new, empty journal under the new name before
+          # this method ever got a chance to run — and the ALTER TABLE ...
+          # RENAME below would then fail, renaming onto a name that
+          # already exists.
+          #
+          # Three-way precheck, cheapest first:
+          #   1. no hecks_eras table at all yet — a genuinely fresh
+          #      database; nothing to bridge, fall through to the
+          #      ordinary provisioning path.
+          #   2. hecks_eras already has rows under the NEW name — this
+          #      rename already ran (a prior boot, possibly this one on a
+          #      retry); idempotent no-op.
+          #   3. hecks_eras has rows under the OLD name — run the
+          #      migration.
+          # Anything else (no rows under either name) falls through
+          # harmlessly — `formerly_known_as` pointing at a name with no
+          # held history is not an error, just inert.
+          def rename_domain!
+            return unless @db.exec_params("SELECT to_regclass($1)", ["hecks_eras"])[0]["to_regclass"]
+            return if @db.exec_params(
+              "SELECT 1 FROM hecks_eras WHERE domain = $1 LIMIT 1", [@domain]
+            ).ntuples.positive?
+            return if @db.exec_params(
+              "SELECT 1 FROM hecks_eras WHERE domain = $1 LIMIT 1", [@formerly_known_as]
+            ).ntuples.zero?
+
+            old_journal  = "hecks_journal_#{Naming.snake(@formerly_known_as)}"
+            old_sequence = "#{old_journal}_ordinal"
+            ordinals = @db.exec_params(
+              "SELECT ordinal FROM hecks_eras WHERE domain = $1 ORDER BY ordinal", [@formerly_known_as]
+            ).map { |row| row["ordinal"].to_i }
+
+            @db.exec("BEGIN")
+            @db.exec("SET LOCAL lock_timeout = '10s'")
+            # Fixed order — old before new, the `eras:` family before the
+            # `ordinal:` family — so two rename attempts (or a rename
+            # racing a mint/plain-write on either name) can only ever
+            # queue behind one another, never deadlock.
+            [
+              "hecks_eras:#{@formerly_known_as}", "hecks_eras:#{@domain}",
+              "hecks_ordinal:#{@formerly_known_as}", "hecks_ordinal:#{@domain}"
+            ].each do |key|
+              @db.exec_params("SELECT pg_advisory_xact_lock(hashtext($1))", [key])
+            end
+
+            @db.exec("ALTER TABLE #{quote(old_journal)} RENAME TO #{quote(journal)}")
+            # An owned SERIAL/IDENTITY sequence moves automatically with
+            # its table and errors if renamed explicitly — this one is a
+            # plain CREATE SEQUENCE the journal's DEFAULT merely points
+            # at (see ensure_base!), so it does need its own rename, and
+            # the column default survives untouched: Postgres stores
+            # nextval('...') as a regclass reference internally, not
+            # literal text.
+            @db.exec("ALTER SEQUENCE #{quote(old_sequence)} RENAME TO #{quote(sequence)}")
+            # ALTER TABLE ... RENAME on the parent does NOT cascade to
+            # child partition names — each one needs its own explicit
+            # rename, sourced from the ordinals held under the OLD name,
+            # captured above before the UPDATE below flips the column.
+            ordinals.each do |ordinal|
+              old_partition = "#{old_journal}_era_#{ordinal}"
+              @db.exec("ALTER TABLE #{quote(old_partition)} RENAME TO #{quote(partition(ordinal))}")
+            end
+
+            %w[hecks_eras hecks_era_texts hecks_approvals].each do |table|
+              @db.exec_params("UPDATE #{table} SET domain = $1 WHERE domain = $2", [@domain, @formerly_known_as])
+            end
+            # Unlike its siblings, hecks_attestations is not created in
+            # ensure_base! at all — only lazily, on first reattest! — so a
+            # domain that never needed one must not be forced through an
+            # UPDATE against a table that doesn't exist.
+            @db.exec_params("UPDATE hecks_attestations SET domain = $1 WHERE domain = $2", [@domain, @formerly_known_as]) if @db.exec_params("SELECT to_regclass($1)", ["hecks_attestations"])[0]["to_regclass"]
+
+            @db.exec("COMMIT")
+          rescue PG::LockNotAvailable
+            @db.exec("ROLLBACK") rescue nil
+            raise Runtime::WiringError,
+                  "cannot rename #{@formerly_known_as} to #{@domain}: another rename, mint, or write holds " \
+                  "one of the domain locks — waited 10s; try again shortly"
+          rescue PG::Error => error
+            @db.exec("ROLLBACK") rescue nil
+            raise Runtime::WiringError, "cannot rename #{@formerly_known_as} to #{@domain}: #{error.message.strip}"
+          end
+
+          # Nothing provisioned yet — build it. Provisioned and owned —
+          # keep it current. Provisioned by SOMEONE ELSE — this is an app
+          # role, and the owner has already done this work.
+          def provisioner?
+            rows = @db.exec_params(
+              "SELECT pg_get_userbyid(relowner) = current_user AS owned FROM pg_class " \
+              "WHERE relname = $1 AND pg_table_is_visible(oid)",
+              [journal]
+            )
+            rows.ntuples.zero? || rows[0]["owned"] == "t"
+          end
+
+          # BUILD, THEN ATTACH — never CREATE ... PARTITION OF. The two
+          # produce the same partition; only the lock differs, and that
+          # difference is the whole availability story of a mint:
+          #
+          #   CREATE TABLE ... PARTITION OF  → AccessExclusiveLock (parent)
+          #   CREATE, then ALTER ... ATTACH  → ShareUpdateExclusiveLock
+          #
+          # AccessExclusive conflicts with every insert in the hierarchy —
+          # routed through the parent OR addressed to an existing leaf —
+          # so attaching the new era inside the mint transaction stopped
+          # every writer for the WHOLE mint, tail materialization
+          # included. ShareUpdateExclusive conflicts with neither, so the
+          # old checkout keeps writing its own era straight through the
+          # build and only pauses for the head swap at the end.
+          #
+          # That is what makes the fork real DURING a mint rather than
+          # merely before and after one. Measured, and pinned by the spec
+          # — which writes through a live mint rather than reading a lock
+          # mode out of the catalog.
+          def ensure_partition!(era)
+            return if partition_attached?(era)
+
+            @db.exec(<<~SQL)
+              CREATE TABLE IF NOT EXISTS #{quote(partition(era))} (
+                LIKE #{quoted_journal} INCLUDING DEFAULTS
+              )
+            SQL
+            @db.exec(<<~SQL)
+              ALTER TABLE #{quoted_journal}
+                ATTACH PARTITION #{quote(partition(era))} FOR VALUES IN (#{era.to_i})
+            SQL
+          end
+
+          # Attached, not merely present: a crash between the CREATE and
+          # the ATTACH leaves a table that is not yet part of the journal,
+          # and the next boot must finish the job rather than skip it.
+          def partition_attached?(era)
+            @db.exec_params(
+              "SELECT 1 FROM pg_inherits i " \
+              "JOIN pg_class child ON child.oid = i.inhrelid " \
+              "JOIN pg_class parent ON parent.oid = i.inhparent " \
+              "WHERE child.relname = $1 AND parent.relname = $2 " \
+              "AND pg_table_is_visible(child.oid) AND pg_table_is_visible(parent.oid)",
+              [partition(era), journal]
+            ).ntuples.positive?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hecks/ports/persistence/plugins/era/postgres_era/lineage/resumable_backfill.rb

@@ -0,0 +1,168 @@
+module Hecks
+  module Adapters
+    class PostgresEra
+      class Lineage
+        # THE ONE CHUNKED, LOCK-FREE, RESUMABLE BACKFILL LOOP — shared by
+        # `backfill_head_snapshot!` (era 1's existing one-shot blocking
+        # backfill, retrofit) and every field-cache table's own initial
+        # backfill (new). Governing principle 1 (docs/implemented/postgres-era-adapter-
+        # split-plan.md): no operation this plan touches may hold a lock
+        # across a scan whose duration scales with table size — a single
+        # `INSERT ... SELECT` over the whole journal (what
+        # `backfill_head_snapshot!` used to be) is exactly that, and so is
+        # a naive "populate every cache row in one statement" field-cache
+        # backfill.
+        #
+        # THE SHAPE: read one bounded chunk (real rows, real ordinals) with
+        # a plain SELECT — no lock held across it, so an ordinary reader or
+        # writer is never blocked by a backfill in progress — then upsert
+        # that chunk under the SAME transactionally-scoped advisory lock +
+        # ordinal-guard idiom `append`'s own snapshot upsert already uses
+        # (`WHERE ordinal < EXCLUDED.ordinal`), then persist a cursor
+        # before moving to the next chunk. Repeat until a chunk reads back
+        # short of a full page — that page was the last one.
+        #
+        # RESUMABLE, not merely restartable. A crash (or a second
+        # concurrent boot) mid-backfill leaves the cursor exactly where the
+        # last COMMITTED chunk left it — `hecks_backfill_progress` is
+        # updated in the SAME transaction as the chunk's own upsert, so
+        # cursor and data can never observably disagree (see
+        # `run_chunk!`). The next attempt reads that cursor and continues;
+        # it does not rescan what a prior attempt already committed.
+        # RESTARTABLE would also be CORRECT here (every upsert is
+        # idempotent and ordinal-guarded — rerunning an already-done chunk
+        # from id 1 changes nothing) but wastes real work on a large
+        # table; resumability is what keeps a crash near the END of a
+        # large backfill cheap to recover from instead of starting over.
+        #
+        # THE LOCK KEY PREFIX is `hecks_field_cache:` — deliberately
+        # disjoint from the three families already in use elsewhere in
+        # this adapter (`hecks_ordinal:`, `hecks_eras:`,
+        # `hecks_head_snapshot:` — see lineage.rb/head_compiler.rb/
+        # mint_transaction.rb/tail_merge.rb) so a backfill chunk NEVER
+        # contends with a plain write, a mint, or a snapshot-table's own
+        # first-creation lock. It is held for exactly ONE CHUNK's own
+        # transaction, never across the whole backfill — two concurrent
+        # backfillers of the SAME target simply take turns one chunk at a
+        # time rather than racing to duplicate work; neither blocks an
+        # unrelated reader or writer for even an instant.
+        module ResumableBackfill
+          CHUNK_SIZE = 5_000
+
+          # Idempotent, unguarded — same idiom as every other DDL helper
+          # in this file tree (`ensure_head_snapshot!` et al.): cheap,
+          # runs on every boot, only ever does real work once.
+          def ensure_backfill_progress_table!
+            @db.exec(<<~SQL)
+              CREATE TABLE IF NOT EXISTS hecks_backfill_progress (
+                target     text PRIMARY KEY,
+                cursor     text,
+                completed  boolean NOT NULL DEFAULT false,
+                updated_at timestamptz NOT NULL DEFAULT now()
+              )
+            SQL
+          end
+
+          # Drives `target` (an already-created, currently-empty-or-
+          # partially-filled table) through chunks until a source read
+          # comes back short of `CHUNK_SIZE` rows. Two distinct callables,
+          # not one — a head-snapshot row and a field-cache row carry
+          # different columns (`state` jsonb vs. a single extracted
+          # `value`), so there is no one generic "upsert this row" shape
+          # to share; only the LOOP, the lock, and the cursor are generic.
+          #
+          #   source_sql.call(cursor) — given the last-processed id (nil
+          #     before the first chunk), returns a SQL SELECT whose result
+          #     has an `id` column (text, ordered ascending) plus whatever
+          #     other columns `upsert` below needs. Must read `id >
+          #     cursor` (or unconditional when cursor is nil), `ORDER BY
+          #     id`, `LIMIT CHUNK_SIZE` — the caller owns the actual
+          #     column list/source tables; this method only owns the loop,
+          #     the lock, and the cursor.
+          #
+          #   upsert.call(rows) — given the PG::Result of one chunk's
+          #     read, performs the actual guarded upsert into `target` and
+          #     returns nothing meaningful; runs INSIDE the same
+          #     transaction/advisory-lock scope as the cursor update below,
+          #     so a crash between "wrote the chunk" and "advanced the
+          #     cursor" is impossible — they commit together or not at
+          #     all.
+          def chunked_backfill!(target, source_sql:, upsert:)
+            ensure_backfill_progress_table!
+            loop do
+              done = run_chunk!(target, source_sql: source_sql, upsert: upsert)
+              break if done
+            end
+          end
+
+          private
+
+          # ONE CHUNK, ONE TRANSACTION, ONE SHORT-HELD LOCK. Re-reads
+          # progress AFTER acquiring the lock (not just before) — a second
+          # concurrent booter may have already finished this exact chunk
+          # (or the whole backfill) while this process was waiting for the
+          # lock; without the re-read, it would redundantly reprocess a
+          # chunk another process just committed. Harmless either way
+          # (idempotent, ordinal-guarded) but the re-read is what keeps
+          # two concurrent boots from both doing the full scan instead of
+          # splitting it.
+          def run_chunk!(target, source_sql:, upsert:)
+            progress = backfill_progress(target)
+            return true if progress[:completed]
+
+            completed = false
+            nested_transaction("hecks_backfill_chunk") do
+              @db.exec_params("SELECT pg_advisory_xact_lock(hashtext('hecks_field_cache:' || $1))", [target])
+              progress = backfill_progress(target)
+              if progress[:completed]
+                completed = true
+                next
+              end
+
+              rows = @db.exec(source_sql.call(progress[:cursor]))
+              if rows.ntuples.zero?
+                upsert_backfill_progress!(target, cursor: progress[:cursor], completed: true)
+                completed = true
+                next
+              end
+
+              upsert.call(rows)
+              completed = rows.ntuples < CHUNK_SIZE
+              # `PG::Result#[]` supports neither an out-of-range index NOR
+              # a negative one (unlike a plain Ruby Array) — same trap as
+              # `backfill_progress` above, the explicit last-index form.
+              last_cursor = rows[rows.ntuples - 1]["id"]
+              upsert_backfill_progress!(target, cursor: last_cursor, completed: completed)
+            end
+            completed
+          end
+
+          # `PG::Result#[]` RAISES IndexError on an out-of-range index —
+          # unlike a plain Ruby Array, it does not return nil — so the
+          # ENTIRELY ORDINARY case of "no progress row exists yet" (every
+          # target's very first check) cannot be read via a bare `[0]`.
+          # `ntuples.zero?` first, always.
+          def backfill_progress(target)
+            result = @db.exec_params(
+              "SELECT cursor, completed FROM hecks_backfill_progress WHERE target = $1", [target]
+            )
+            return { cursor: nil, completed: false } if result.ntuples.zero?
+
+            row = result[0]
+            { cursor: row["cursor"], completed: row["completed"] == "t" }
+          end
+
+          def upsert_backfill_progress!(target, cursor:, completed:)
+            @db.exec_params(
+              "INSERT INTO hecks_backfill_progress (target, cursor, completed, updated_at) " \
+              "VALUES ($1, $2, $3, now()) " \
+              "ON CONFLICT (target) DO UPDATE SET cursor = EXCLUDED.cursor, " \
+              "completed = EXCLUDED.completed, updated_at = EXCLUDED.updated_at",
+              [target, cursor, completed]
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hecks/ports/persistence/plugins/era/postgres_era/lineage/tail_merge.rb

@@ -0,0 +1,170 @@
+require_relative "../../../../../../runtime/registry"
+
+module Hecks
+  module Adapters
+    class PostgresEra
+      class Lineage
+        module TailMerge
+          # ── fork observability ─────────────────────────────────────────
+
+          # Post-cut writes an old era made after the newer era was minted
+          # — the divergence between the worlds, observable at any time.
+          def diverged_count(old_era)
+            cut = eras.find { |era| era[:ordinal] == old_era + 1 }&.dig(:watermark)
+            return 0 unless cut
+
+            @db.exec(
+              "SELECT count(*) FROM #{quoted_journal} WHERE era = #{old_era.to_i} AND ordinal > #{cut.to_i}"
+            )[0]["count"].to_i
+          end
+
+          # ── tail-merge ─────────────────────────────────────────────────
+          #
+          # The one deliberate command — it marks a business event (an app
+          # retiring), never a shape change. One transaction: advance the
+          # watermarks, rebuild the head so the tail interleaves by its
+          # recorded global ordinal, append the declared winners, audit —
+          # and roll the whole thing back on any refusal. Records touched
+          # by both worlds since the cut refuse until each has an explicit
+          # winner; resolution itself is append-only (the winner's state
+          # re-enters as the newest row and wins structurally — originals
+          # stay immutable).
+          def merge_tail!(aggregates:, edges:, winners: {}, audit: nil)
+            @db.exec("BEGIN")
+            @db.exec("SET LOCAL lock_timeout = '10s'")
+            @db.exec("SELECT pg_advisory_xact_lock(hashtext('hecks_eras:' || #{text_literal(@domain)}))")
+            held = eras
+            era = held.last[:ordinal]
+            label = held.last[:label]
+            if era == 1
+              @db.exec("ROLLBACK")
+              raise Runtime::WiringError, "nothing to merge — #{@domain} stands at era 1"
+            end
+            cut = held.last[:watermark].to_i
+            tip = last_ordinal
+
+            conflicts = aggregates.flat_map { |aggregate| conflict_ids(aggregate, edges, era, cut) }
+            unresolved = conflicts.reject { |_, id| winners.key?(id) }
+            unless unresolved.empty?
+              @db.exec("ROLLBACK")
+              raise Runtime::WiringError,
+                    "cannot merge the tail of #{@domain}: touched by both worlds since the cut — " \
+                    "#{unresolved.map { |storage, id| "#{storage}##{id}" }.sort.join(', ')}. " \
+                    "Name each winner (--winner <id>=old or --winner <id>=new), then run bin/merge_tail again. " \
+                    "A winner takes the WHOLE record — the aggregate is the consistency boundary, so the " \
+                    "loser's edits are discarded even where they touched different attributes"
+            end
+
+            # the new world's pre-merge head states, captured before the
+            # rebuild lets the tail interleave
+            new_states = {}
+            aggregates.each do |aggregate|
+              winners.select { |_, side| side == "new" }.each_key do |id|
+                row = @db.exec_params("SELECT state FROM #{quote(head_view(aggregate.storage_name))} WHERE id = $1", [id])
+                new_states[id] = [aggregate.storage_name, row[0]["state"]] if row.ntuples.positive?
+              end
+            end
+
+            @db.exec_params("UPDATE hecks_eras SET watermark = $2 WHERE domain = $1 AND ordinal > 1", [@domain, tip])
+            aggregates.each do |aggregate|
+              @db.exec("DROP VIEW IF EXISTS #{quote(head_view(aggregate.storage_name))}")
+              @db.exec("DROP MATERIALIZED VIEW IF EXISTS #{quote(matview(aggregate.storage_name, era, label))}")
+              # full: the watermarks just moved — every ancestor matview's
+              # cut is stale, so there is nothing safe to layer on.
+              compile_head!(aggregate, era, label, edges, full: true)
+            end
+
+            # rubocop:disable-next Metrics/BlockLength
+            winners.each do |id, side|
+              aggregates.each do |aggregate|
+                state =
+                  if side == "old"
+                    row = @db.exec_params(
+                      "SELECT state FROM #{quote(matview(aggregate.storage_name, era, label))} " \
+                      "WHERE aggregate_id = $1 AND operation = 'save' ORDER BY ordinal DESC LIMIT 1",
+                      [id]
+                    )
+                    row.ntuples.positive? ? row[0]["state"] : nil
+                  else
+                    new_states[id]&.first == aggregate.storage_name ? new_states[id][1] : nil
+                  end
+                next unless state
+
+                # Same two-step append does for a live write — journal
+                # first, snapshot second — because this INSERT bypasses
+                # PostgresEra#append entirely (it writes through Lineage
+                # directly). Skipping the snapshot half here would mean a
+                # merge winner lands in the journal but head_view — which
+                # reads this era's live rows from the snapshot table, not
+                # by re-scanning the journal — never shows it.
+                ordinal = @db.exec_params(
+                  "INSERT INTO #{quoted_journal} (era, aggregate, aggregate_id, operation, state) " \
+                  "VALUES ($1, $2, $3, 'save', $4) RETURNING ordinal",
+                  [era, aggregate.storage_name, id, state]
+                )[0]["ordinal"]
+                # Always `'save'` — a merge winner is read from either
+                # the matview's own `operation = 'save'` rows or the new
+                # world's live head (`new_states`, captured from
+                # `head_view`, which is save-only by construction), so
+                # nothing reaching this INSERT is ever a delete.
+                @db.exec_params(
+                  "INSERT INTO #{quote(head_snapshot(aggregate.storage_name, era))} (id, ordinal, operation, state) " \
+                  "VALUES ($1, $2, 'save', $3) " \
+                  "ON CONFLICT (id) DO UPDATE SET ordinal = EXCLUDED.ordinal, operation = EXCLUDED.operation, " \
+                  "state = EXCLUDED.state WHERE #{quote(head_snapshot(aggregate.storage_name, era))}.ordinal < EXCLUDED.ordinal",
+                  [id, ordinal, state]
+                )
+              end
+            end
+
+            if audit
+              violations = audit.call
+              unless violations.empty?
+                @db.exec("ROLLBACK")
+                raise Runtime::WiringError,
+                      "cannot merge the tail of #{@domain}: the audit refused —\n  - #{violations.join("\n  - ")}"
+              end
+            end
+
+            @db.exec("COMMIT")
+            true
+          rescue PG::LockNotAvailable
+            @db.exec("ROLLBACK") rescue nil
+            raise Runtime::WiringError,
+                  "cannot merge the tail of #{@domain}: another mint or merge holds the domain lock — " \
+                  "waited 10s; try again shortly"
+          rescue PG::Error => error
+            @db.exec("ROLLBACK") rescue nil
+            raise Runtime::WiringError, "cannot merge the tail of #{@domain}: #{error.message.strip}"
+          end
+
+          # Ids touched by BOTH worlds since the cut — the old world's
+          # post-cut tail INTERSECTed with the new world's own writes.
+          #
+          # KNOWN GAP, not silently risked: this compares raw
+          # `aggregate_id` values, with no notion of "these two different
+          # ids are the same entity, rekeyed." If a domain's history
+          # includes a rekey (see TranslationRekey) and is LATER
+          # merged here, a record's pre-rekey and post-rekey rows will
+          # never intersect — they just silently survive as two separate,
+          # unrelated-looking heads (a duplicate, not corruption: nothing
+          # here deletes or clobbers either side). Resolve any such
+          # duplicate manually after a merge; teaching this INTERSECT
+          # about a rekey mapping is real, separate work, deliberately
+          # out of scope for rekey's first pass.
+          def conflict_ids(aggregate, edges, era, cut)
+            names = names_by_era(aggregate, edges)
+            olds = (1...era).map { |ancestor| text_literal(names[:storage][ancestor - 1]) }.join(", ")
+            @db.exec(<<~SQL).map { |row| [aggregate.storage_name, row["aggregate_id"]] }
+              SELECT aggregate_id FROM #{quoted_journal}
+              WHERE era < #{era.to_i} AND aggregate IN (#{olds}) AND ordinal > #{cut.to_i}
+              INTERSECT
+              SELECT aggregate_id FROM #{quoted_journal}
+              WHERE era = #{era.to_i} AND aggregate = #{text_literal(names[:storage][era - 1])}
+            SQL
+          end
+        end
+      end
+    end
+  end
+end