hecks 0.2.0 → 0.3.0

data/lib/hecks/adapters/driven/postgres_era/lineage/mint_transaction.rb

@@ -0,0 +1,166 @@
+require_relative "../../../../runtime/registry"
+require_relative "../../../../runtime/storage_shape"
+
+module Hecks
+  module Adapters
+    class PostgresEra
+      class Lineage
+        module MintTransaction
+          # ── the mint transaction ───────────────────────────────────────
+          #
+          # One transaction: the new era row (held text + minted name +
+          # cut watermark), the new partition, and every aggregate's
+          # recompiled matview + head view. The advisory lock is the
+          # writer fence — two concurrent minters serialize, and the
+          # second finds the era already held. Populating the matview
+          # inside the transaction means a convert meeting an unmapped
+          # value REFUSES the whole mint, loudly, before anything boots.
+          def mint_era!(ordinal:, hash:, label:, held_text:, aggregates:, edges:, role: nil, projection: nil)
+            @db.exec("BEGIN")
+            # A concurrent minter blocks briefly, then refuses with a name
+            # rather than hanging: mint is non-interactive (the human
+            # decision already happened in the audit tool), so the lock is
+            # only ever held for the transaction itself.
+            @db.exec("SET LOCAL lock_timeout = '10s'")
+            @db.exec("SELECT pg_advisory_xact_lock(hashtext('hecks_eras:' || #{text_literal(@domain)}))")
+            if eras.any? { |era| era[:ordinal] == ordinal }
+              @db.exec("ROLLBACK")
+              return false
+            end
+
+            watermark = last_ordinal
+            @db.exec_params(
+              "INSERT INTO hecks_eras (domain, ordinal, hash, label, held_text, watermark, held_digest, held_projection, canon_form) " \
+              "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
+              [@domain, ordinal, hash, label, held_text, watermark, Digest::SHA256.hexdigest(held_text),
+               projection && JSON.generate(projection), Runtime::StorageShape::FORM_VERSION]
+            )
+            archive_text!(ordinal, held_text)
+            ensure_partition!(ordinal)
+            # AFTER compile_head!, not before — this era's own snapshot
+            # table (a possibly-renamed name, per aggregate) is what
+            # grant_role! grants on, and compile_head! is what creates it.
+            # Granting first would GRANT on a relation that does not
+            # exist yet for any aggregate renamed in this very edge.
+            aggregates.each { |aggregate| compile_head!(aggregate, ordinal, label, edges) }
+            grant_role!(role, aggregates: aggregates, era: ordinal) if role
+            # LAST, right before COMMIT — not merely unconditional. Once
+            # acquired, a lock is held until the TRANSACTION ends, not
+            # just for the statement that took it — so advance_era!'s
+            # DROP POLICY/CREATE POLICY (AccessExclusiveLock, same family
+            # as ALTER TABLE, and unavoidably so: Postgres has no lighter
+            # form for changing a policy, unlike the partition attach
+            # below) blocks every concurrent writer for as long as it sits
+            # BEFORE the expensive step. Ordered here, that block is the
+            # width of a few catalog statements plus the commit itself,
+            # not the width of compile_head!'s matview build. Measured:
+            # moving this above compile_head! (an earlier ordering, caught
+            # only once a genuine concurrent-write test was built rather
+            # than assumed) reintroduced exactly the mint-stops-the-world
+            # cost ensure_partition!'s build-then-ATTACH exists to avoid.
+            #
+            # UNCONDITIONAL regardless of position — this is the line that
+            # drops writing to the old schema. It does not wait for a role
+            # to be configured on THIS boot, because the cutoff is a fact
+            # about the era, not about who happened to mint it: an old
+            # checkout's role, granted by some earlier boot this one knows
+            # nothing about, must lose write access the instant this
+            # transaction commits.
+            advance_era!(ordinal)
+            @db.exec("COMMIT")
+            true
+          rescue PG::LockNotAvailable
+            @db.exec("ROLLBACK") rescue nil
+            raise Runtime::WiringError,
+                  "cannot mint era #{ordinal} of #{@domain}: another mint holds the domain lock — " \
+                  "waited 10s; try again shortly"
+          rescue PG::Error => error
+            @db.exec("ROLLBACK") rescue nil
+            raise Runtime::WiringError, "cannot mint era #{ordinal} of #{@domain}: #{error.message.strip}"
+          end
+
+          # Base privileges for a deployment's app role — a NON-owner,
+          # which may append and read once the shared era fence below
+          # admits it, and owns nothing. Idempotent, and unconcerned with
+          # WHICH era is current: that is advance_era!'s job, not this
+          # one's, so a role can be onboarded at any time without
+          # disturbing who may write what right now.
+          #
+          # Owner-only work, so a non-owner app boot skips it: the
+          # provisioner has already granted that role, and re-affirming is
+          # idempotent anyway.
+          #
+          # `aggregates:`/`era:` cover the read-cache side of the same
+          # story: unlike the journal (immutable, owner-provisioned once),
+          # each aggregate's head_snapshot table is a table an app role
+          # must itself INSERT/UPDATE/DELETE into — PostgresEra#append writes
+          # it directly, not through a view — so it needs real DML grants,
+          # not just the SELECT a derived read surface would need. `era:`
+          # is the ordinal THIS role is about to write under (the one
+          # hold_first!/mint_era! just made current, or the superseded one
+          # a stale checkout still speaks) — head_snapshot is era-scoped,
+          # so granting on the wrong era's table would grant on a table
+          # this role will never touch. Every caller passes its own
+          # bluebook's aggregates and resolved era — including a
+          # held-but-superseded checkout, since a role connecting to it
+          # for the first time (a new instance of an old checkout) has no
+          # privileges yet either.
+          def grant_role!(role, aggregates: [], era: nil)
+            return unless provisioner?
+
+            quoted_role = quote(role)
+            @db.exec("GRANT INSERT, SELECT ON #{quoted_journal} TO #{quoted_role}")
+            @db.exec("GRANT USAGE ON SEQUENCE #{quote(sequence)} TO #{quoted_role}")
+            return unless era
+
+            aggregates.each do |aggregate|
+              storage_name = aggregate.storage_name
+              @db.exec("GRANT SELECT, INSERT, UPDATE, DELETE ON #{quote(head_snapshot(storage_name, era))} TO #{quoted_role}")
+              @db.exec("GRANT SELECT ON #{quote(head_view(storage_name))} TO #{quoted_role}") if view_exists?(head_view(storage_name))
+            end
+          end
+
+          # THE current-era fence — ONE policy, shared by every granted
+          # role, not one per role. Advancing it is what drops writing to
+          # the old schema the instant the new one materializes: the
+          # moment this commits, no role — old or new, whether or not it
+          # did anything to earn this mint — may insert anything but the
+          # era named here. There is no persisted fork: a checkout that
+          # keeps a role fenced to a stale era does not exist in this
+          # design, because there is no such thing as a role fenced to an
+          # era at all — only the ONE era everyone currently shares.
+          #
+          # A ROW POLICY, not a partition grant. Postgres checks INSERT
+          # privilege on the partitioned PARENT for a routed insert and
+          # never consults the partition's own grants, so a per-partition
+          # GRANT/REVOKE is inert in both directions: grant only on the
+          # partition and nobody can write at all; grant on the parent and
+          # they may write into EVERY era, ancestors included. Measured,
+          # not reasoned about — see the spec, which writes through the
+          # fence rather than asserting the catalog.
+          #
+          # The table owner bypasses RLS by default, and that is load
+          # bearing: mint and merge run as the owner and must be able to
+          # write any era (the merge re-enters a winner's state into the
+          # CURRENT era, and compile_head! reads every ancestor).
+          #
+          # CALL ONLY WITH THE NEW CURRENT ORDINAL — from hold_first! (era
+          # 1) or mint_era! (era N). Calling this with a SUPERSEDED
+          # ordinal — from a boot that merely recognizes an old checkout —
+          # would roll the fence backward and silently reopen the old
+          # schema for everyone. That path grants a role's PRIVILEGES
+          # (grant_role!) and stops there on purpose.
+          def advance_era!(ordinal)
+            @db.exec("DROP POLICY IF EXISTS hecks_current_era ON #{quoted_journal}")
+            @db.exec(
+              "CREATE POLICY hecks_current_era ON #{quoted_journal} FOR INSERT TO PUBLIC " \
+              "WITH CHECK (era = #{ordinal.to_i})"
+            )
+            @db.exec("DROP POLICY IF EXISTS hecks_read_all ON #{quoted_journal}")
+            @db.exec("CREATE POLICY hecks_read_all ON #{quoted_journal} FOR SELECT TO PUBLIC USING (true)")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hecks/adapters/driven/postgres_era/lineage/provisioning.rb

@@ -0,0 +1,286 @@
+module Hecks
+  module Adapters
+    class PostgresEra
+      class Lineage
+        module Provisioning
+          # Provisioning is the OWNER's job, and a deployment's app role is
+          # deliberately not the owner — it may append and read, and it
+          # owns nothing. By the time such a role connects, the base is
+          # already built, so its boot verifies rather than builds.
+          #
+          # Without this guard the per-era fence below is unreachable: the
+          # ALTER TABLE and REVOKE here are owner-only, so the very role
+          # grant_era! exists to constrain could never finish booting
+          # ("must be owner of table hecks_eras"). A fence nothing can
+          # reach is not a fence.
+          def ensure_base!
+            rename_domain! if @formerly_known_as
+            return unless provisioner?
+
+            @db.exec(<<~SQL)
+              CREATE TABLE IF NOT EXISTS hecks_eras (
+                domain    text NOT NULL,
+                ordinal   int  NOT NULL,
+                hash      text,
+                label     text,
+                held_text text NOT NULL,
+                watermark bigint,
+                PRIMARY KEY (domain, ordinal)
+              )
+            SQL
+            @db.exec("ALTER TABLE hecks_eras ADD COLUMN IF NOT EXISTS held_digest text")
+            @db.exec("ALTER TABLE hecks_eras ADD COLUMN IF NOT EXISTS held_projection jsonb")
+            # which canonical-form version minted this name — see
+            # Runtime::StorageShape::FORM_VERSION; rows minted before the
+            # column carry NULL, read as an implicit 1
+            @db.exec("ALTER TABLE hecks_eras ADD COLUMN IF NOT EXISTS canon_form int")
+            # every frozen text version, archived where an edit cannot
+            # reach it — the recovery the hard reattest refusal points at
+            @db.exec(<<~SQL)
+              CREATE TABLE IF NOT EXISTS hecks_era_texts (
+                domain      text NOT NULL,
+                ordinal     int  NOT NULL,
+                digest      text NOT NULL,
+                held_text   text NOT NULL,
+                archived_at timestamptz NOT NULL DEFAULT now(),
+                PRIMARY KEY (domain, ordinal, digest)
+              )
+            SQL
+            @db.exec(<<~SQL)
+              CREATE TABLE IF NOT EXISTS hecks_approvals (
+                domain           text NOT NULL,
+                from_label       text NOT NULL,
+                to_label         text NOT NULL,
+                edge_digest      text NOT NULL,
+                reviewed_ordinal bigint NOT NULL,
+                approved_at      timestamptz NOT NULL DEFAULT now()
+              )
+            SQL
+            @db.exec("CREATE SEQUENCE IF NOT EXISTS #{quote(sequence)}")
+            # GENERATED ALWAYS AS IDENTITY is the intent, but identity
+            # columns on partitioned tables need Postgres 17 — an owned
+            # sequence default is the same spanning ordinal on any
+            # supported server.
+            @db.exec(<<~SQL)
+              CREATE TABLE IF NOT EXISTS #{quoted_journal} (
+                ordinal      bigint NOT NULL DEFAULT nextval('#{sequence}'),
+                era          int    NOT NULL,
+                aggregate    text   NOT NULL,
+                aggregate_id text   NOT NULL,
+                operation    text   NOT NULL DEFAULT 'save',
+                state        jsonb,
+                mirrors      jsonb
+              ) PARTITION BY LIST (era)
+            SQL
+            ensure_partition!(1)
+            # Immutability by privilege: nothing updates or deletes journal
+            # rows. The owner's implicit rights remain (Postgres has no way
+            # to revoke them from the owner itself); a deployment's app
+            # role connects as a NON-owner and gets exactly INSERT, per
+            # era, at mint time.
+            @db.exec("REVOKE UPDATE, DELETE ON #{quoted_journal} FROM PUBLIC")
+            # RLS goes on AT PROVISIONING, never mid-life — enabling it
+            # later would deny every role that has no policy yet, on
+            # whatever the shape of the schema happened to be at that
+            # moment.
+            #
+            # FORCE, not merely ENABLE: without it, the table OWNER is
+            # exempt from every policy here, by Postgres default — which
+            # would leave the schema writable forever to whoever holds
+            # the owner's credentials, the one connection this whole
+            # design cannot fence. Checked, not assumed: mint_era! never
+            # inserts into the journal at all (only hecks_eras/
+            # hecks_era_texts, neither RLS-protected), and merge_tail!'s
+            # one journal INSERT targets the CURRENT era, which the fence
+            # already admits for anyone with base privileges — so FORCE
+            # costs the owner nothing operations here actually need.
+            #
+            # This still exempts an actual Postgres SUPERUSER (or any
+            # role granted BYPASSRLS) unconditionally — FORCE only
+            # narrows what ENABLE already narrows for the owner
+            # specifically, and superuser bypass sits above both. Running
+            # migrations as a real superuser (self-hosted Postgres, most
+            # commonly) leaves this gap open regardless; a managed
+            # provider's admin account is typically NOT a superuser, and
+            # is exactly what FORCE closes.
+            #
+            # GUARDED, not reissued unconditionally — measured, not
+            # assumed: `ALTER TABLE ... ENABLE/FORCE ROW LEVEL SECURITY`
+            # takes AccessExclusiveLock EVEN WHEN THE SETTING IS ALREADY
+            # CORRECT (Postgres does not skip the lock just because the
+            # statement would be a no-op). ensure_base! runs on EVERY
+            # boot by the owning role, not only the first — so an
+            # unconditional reissue here would mean every ordinary
+            # reboot of the deployment's own identity re-freezes every
+            # concurrent writer, on any era, for as long as that ALTER
+            # TABLE has to wait its turn. Read the current state first;
+            # touch the catalog only on the boot that actually needs to.
+            # pg_table_is_visible, NOT a bare relname match — a shared
+            # instance (storehouse) can hold a same-named journal table
+            # per schema; catalog lookups here must resolve the SAME way
+            # search_path resolves an unqualified SQL reference, or a
+            # sibling domain's table satisfies a query meant for this
+            # domain's own.
+            current = @db.exec_params(
+              "SELECT relrowsecurity, relforcerowsecurity FROM pg_class " \
+              "WHERE relname = $1 AND pg_table_is_visible(oid)", [journal]
+            )[0]
+            @db.exec("ALTER TABLE #{quoted_journal} ENABLE ROW LEVEL SECURITY") unless current["relrowsecurity"] == "t"
+            @db.exec("ALTER TABLE #{quoted_journal} FORCE ROW LEVEL SECURITY") unless current["relforcerowsecurity"] == "t"
+            install_transforms!
+          end
+
+          # A DOMAIN'S OWN IDENTITY CHANGED — bridge its history under the
+          # new name, before `provisioner?`/the CREATE TABLE IF NOT EXISTS
+          # block below ever run. That ordering is load-bearing, not
+          # tidiness: `provisioner?` and every statement in ensure_base!
+          # test for the journal under `journal` — the NEW name — and a
+          # freshly-renamed domain looks, to those checks, exactly like a
+          # domain that has never been provisioned at all. Left where it
+          # was written, ensure_base! would happily CREATE TABLE IF NOT
+          # EXISTS a brand-new, empty journal under the new name before
+          # this method ever got a chance to run — and the ALTER TABLE ...
+          # RENAME below would then fail, renaming onto a name that
+          # already exists.
+          #
+          # Three-way precheck, cheapest first:
+          #   1. no hecks_eras table at all yet — a genuinely fresh
+          #      database; nothing to bridge, fall through to the
+          #      ordinary provisioning path.
+          #   2. hecks_eras already has rows under the NEW name — this
+          #      rename already ran (a prior boot, possibly this one on a
+          #      retry); idempotent no-op.
+          #   3. hecks_eras has rows under the OLD name — run the
+          #      migration.
+          # Anything else (no rows under either name) falls through
+          # harmlessly — `formerly_known_as` pointing at a name with no
+          # held history is not an error, just inert.
+          def rename_domain!
+            return unless @db.exec_params("SELECT to_regclass($1)", ["hecks_eras"])[0]["to_regclass"]
+            return if @db.exec_params(
+              "SELECT 1 FROM hecks_eras WHERE domain = $1 LIMIT 1", [@domain]
+            ).ntuples.positive?
+            return if @db.exec_params(
+              "SELECT 1 FROM hecks_eras WHERE domain = $1 LIMIT 1", [@formerly_known_as]
+            ).ntuples.zero?
+
+            old_journal  = "hecks_journal_#{Naming.snake(@formerly_known_as)}"
+            old_sequence = "#{old_journal}_ordinal"
+            ordinals = @db.exec_params(
+              "SELECT ordinal FROM hecks_eras WHERE domain = $1 ORDER BY ordinal", [@formerly_known_as]
+            ).map { |row| row["ordinal"].to_i }
+
+            @db.exec("BEGIN")
+            @db.exec("SET LOCAL lock_timeout = '10s'")
+            # Fixed order — old before new, the `eras:` family before the
+            # `ordinal:` family — so two rename attempts (or a rename
+            # racing a mint/plain-write on either name) can only ever
+            # queue behind one another, never deadlock.
+            [
+              "hecks_eras:#{@formerly_known_as}", "hecks_eras:#{@domain}",
+              "hecks_ordinal:#{@formerly_known_as}", "hecks_ordinal:#{@domain}"
+            ].each do |key|
+              @db.exec_params("SELECT pg_advisory_xact_lock(hashtext($1))", [key])
+            end
+
+            @db.exec("ALTER TABLE #{quote(old_journal)} RENAME TO #{quote(journal)}")
+            # An owned SERIAL/IDENTITY sequence moves automatically with
+            # its table and errors if renamed explicitly — this one is a
+            # plain CREATE SEQUENCE the journal's DEFAULT merely points
+            # at (see ensure_base!), so it does need its own rename, and
+            # the column default survives untouched: Postgres stores
+            # nextval('...') as a regclass reference internally, not
+            # literal text.
+            @db.exec("ALTER SEQUENCE #{quote(old_sequence)} RENAME TO #{quote(sequence)}")
+            # ALTER TABLE ... RENAME on the parent does NOT cascade to
+            # child partition names — each one needs its own explicit
+            # rename, sourced from the ordinals held under the OLD name,
+            # captured above before the UPDATE below flips the column.
+            ordinals.each do |ordinal|
+              old_partition = "#{old_journal}_era_#{ordinal}"
+              @db.exec("ALTER TABLE #{quote(old_partition)} RENAME TO #{quote(partition(ordinal))}")
+            end
+
+            %w[hecks_eras hecks_era_texts hecks_approvals].each do |table|
+              @db.exec_params("UPDATE #{table} SET domain = $1 WHERE domain = $2", [@domain, @formerly_known_as])
+            end
+            # Unlike its siblings, hecks_attestations is not created in
+            # ensure_base! at all — only lazily, on first reattest! — so a
+            # domain that never needed one must not be forced through an
+            # UPDATE against a table that doesn't exist.
+            @db.exec_params("UPDATE hecks_attestations SET domain = $1 WHERE domain = $2", [@domain, @formerly_known_as]) if @db.exec_params("SELECT to_regclass($1)", ["hecks_attestations"])[0]["to_regclass"]
+
+            @db.exec("COMMIT")
+          rescue PG::LockNotAvailable
+            @db.exec("ROLLBACK") rescue nil
+            raise Runtime::WiringError,
+                  "cannot rename #{@formerly_known_as} to #{@domain}: another rename, mint, or write holds " \
+                  "one of the domain locks — waited 10s; try again shortly"
+          rescue PG::Error => error
+            @db.exec("ROLLBACK") rescue nil
+            raise Runtime::WiringError, "cannot rename #{@formerly_known_as} to #{@domain}: #{error.message.strip}"
+          end
+
+          # Nothing provisioned yet — build it. Provisioned and owned —
+          # keep it current. Provisioned by SOMEONE ELSE — this is an app
+          # role, and the owner has already done this work.
+          def provisioner?
+            rows = @db.exec_params(
+              "SELECT pg_get_userbyid(relowner) = current_user AS owned FROM pg_class " \
+              "WHERE relname = $1 AND pg_table_is_visible(oid)",
+              [journal]
+            )
+            rows.ntuples.zero? || rows[0]["owned"] == "t"
+          end
+
+          # BUILD, THEN ATTACH — never CREATE ... PARTITION OF. The two
+          # produce the same partition; only the lock differs, and that
+          # difference is the whole availability story of a mint:
+          #
+          #   CREATE TABLE ... PARTITION OF  → AccessExclusiveLock (parent)
+          #   CREATE, then ALTER ... ATTACH  → ShareUpdateExclusiveLock
+          #
+          # AccessExclusive conflicts with every insert in the hierarchy —
+          # routed through the parent OR addressed to an existing leaf —
+          # so attaching the new era inside the mint transaction stopped
+          # every writer for the WHOLE mint, tail materialization
+          # included. ShareUpdateExclusive conflicts with neither, so the
+          # old checkout keeps writing its own era straight through the
+          # build and only pauses for the head swap at the end.
+          #
+          # That is what makes the fork real DURING a mint rather than
+          # merely before and after one. Measured, and pinned by the spec
+          # — which writes through a live mint rather than reading a lock
+          # mode out of the catalog.
+          def ensure_partition!(era)
+            return if partition_attached?(era)
+
+            @db.exec(<<~SQL)
+              CREATE TABLE IF NOT EXISTS #{quote(partition(era))} (
+                LIKE #{quoted_journal} INCLUDING DEFAULTS
+              )
+            SQL
+            @db.exec(<<~SQL)
+              ALTER TABLE #{quoted_journal}
+                ATTACH PARTITION #{quote(partition(era))} FOR VALUES IN (#{era.to_i})
+            SQL
+          end
+
+          # Attached, not merely present: a crash between the CREATE and
+          # the ATTACH leaves a table that is not yet part of the journal,
+          # and the next boot must finish the job rather than skip it.
+          def partition_attached?(era)
+            @db.exec_params(
+              "SELECT 1 FROM pg_inherits i " \
+              "JOIN pg_class child ON child.oid = i.inhrelid " \
+              "JOIN pg_class parent ON parent.oid = i.inhparent " \
+              "WHERE child.relname = $1 AND parent.relname = $2 " \
+              "AND pg_table_is_visible(child.oid) AND pg_table_is_visible(parent.oid)",
+              [partition(era), journal]
+            ).ntuples.positive?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hecks/adapters/driven/postgres_era/lineage/resumable_backfill.rb

@@ -0,0 +1,168 @@
+module Hecks
+  module Adapters
+    class PostgresEra
+      class Lineage
+        # THE ONE CHUNKED, LOCK-FREE, RESUMABLE BACKFILL LOOP — shared by
+        # `backfill_head_snapshot!` (era 1's existing one-shot blocking
+        # backfill, retrofit) and every field-cache table's own initial
+        # backfill (new). Governing principle 1 (docs/implemented/postgres-era-adapter-
+        # split-plan.md): no operation this plan touches may hold a lock
+        # across a scan whose duration scales with table size — a single
+        # `INSERT ... SELECT` over the whole journal (what
+        # `backfill_head_snapshot!` used to be) is exactly that, and so is
+        # a naive "populate every cache row in one statement" field-cache
+        # backfill.
+        #
+        # THE SHAPE: read one bounded chunk (real rows, real ordinals) with
+        # a plain SELECT — no lock held across it, so an ordinary reader or
+        # writer is never blocked by a backfill in progress — then upsert
+        # that chunk under the SAME transactionally-scoped advisory lock +
+        # ordinal-guard idiom `append`'s own snapshot upsert already uses
+        # (`WHERE ordinal < EXCLUDED.ordinal`), then persist a cursor
+        # before moving to the next chunk. Repeat until a chunk reads back
+        # short of a full page — that page was the last one.
+        #
+        # RESUMABLE, not merely restartable. A crash (or a second
+        # concurrent boot) mid-backfill leaves the cursor exactly where the
+        # last COMMITTED chunk left it — `hecks_backfill_progress` is
+        # updated in the SAME transaction as the chunk's own upsert, so
+        # cursor and data can never observably disagree (see
+        # `run_chunk!`). The next attempt reads that cursor and continues;
+        # it does not rescan what a prior attempt already committed.
+        # RESTARTABLE would also be CORRECT here (every upsert is
+        # idempotent and ordinal-guarded — rerunning an already-done chunk
+        # from id 1 changes nothing) but wastes real work on a large
+        # table; resumability is what keeps a crash near the END of a
+        # large backfill cheap to recover from instead of starting over.
+        #
+        # THE LOCK KEY PREFIX is `hecks_field_cache:` — deliberately
+        # disjoint from the three families already in use elsewhere in
+        # this adapter (`hecks_ordinal:`, `hecks_eras:`,
+        # `hecks_head_snapshot:` — see lineage.rb/head_compiler.rb/
+        # mint_transaction.rb/tail_merge.rb) so a backfill chunk NEVER
+        # contends with a plain write, a mint, or a snapshot-table's own
+        # first-creation lock. It is held for exactly ONE CHUNK's own
+        # transaction, never across the whole backfill — two concurrent
+        # backfillers of the SAME target simply take turns one chunk at a
+        # time rather than racing to duplicate work; neither blocks an
+        # unrelated reader or writer for even an instant.
+        module ResumableBackfill
+          CHUNK_SIZE = 5_000
+
+          # Idempotent, unguarded — same idiom as every other DDL helper
+          # in this file tree (`ensure_head_snapshot!` et al.): cheap,
+          # runs on every boot, only ever does real work once.
+          def ensure_backfill_progress_table!
+            @db.exec(<<~SQL)
+              CREATE TABLE IF NOT EXISTS hecks_backfill_progress (
+                target     text PRIMARY KEY,
+                cursor     text,
+                completed  boolean NOT NULL DEFAULT false,
+                updated_at timestamptz NOT NULL DEFAULT now()
+              )
+            SQL
+          end
+
+          # Drives `target` (an already-created, currently-empty-or-
+          # partially-filled table) through chunks until a source read
+          # comes back short of `CHUNK_SIZE` rows. Two distinct callables,
+          # not one — a head-snapshot row and a field-cache row carry
+          # different columns (`state` jsonb vs. a single extracted
+          # `value`), so there is no one generic "upsert this row" shape
+          # to share; only the LOOP, the lock, and the cursor are generic.
+          #
+          #   source_sql.call(cursor) — given the last-processed id (nil
+          #     before the first chunk), returns a SQL SELECT whose result
+          #     has an `id` column (text, ordered ascending) plus whatever
+          #     other columns `upsert` below needs. Must read `id >
+          #     cursor` (or unconditional when cursor is nil), `ORDER BY
+          #     id`, `LIMIT CHUNK_SIZE` — the caller owns the actual
+          #     column list/source tables; this method only owns the loop,
+          #     the lock, and the cursor.
+          #
+          #   upsert.call(rows) — given the PG::Result of one chunk's
+          #     read, performs the actual guarded upsert into `target` and
+          #     returns nothing meaningful; runs INSIDE the same
+          #     transaction/advisory-lock scope as the cursor update below,
+          #     so a crash between "wrote the chunk" and "advanced the
+          #     cursor" is impossible — they commit together or not at
+          #     all.
+          def chunked_backfill!(target, source_sql:, upsert:)
+            ensure_backfill_progress_table!
+            loop do
+              done = run_chunk!(target, source_sql: source_sql, upsert: upsert)
+              break if done
+            end
+          end
+
+          private
+
+          # ONE CHUNK, ONE TRANSACTION, ONE SHORT-HELD LOCK. Re-reads
+          # progress AFTER acquiring the lock (not just before) — a second
+          # concurrent booter may have already finished this exact chunk
+          # (or the whole backfill) while this process was waiting for the
+          # lock; without the re-read, it would redundantly reprocess a
+          # chunk another process just committed. Harmless either way
+          # (idempotent, ordinal-guarded) but the re-read is what keeps
+          # two concurrent boots from both doing the full scan instead of
+          # splitting it.
+          def run_chunk!(target, source_sql:, upsert:)
+            progress = backfill_progress(target)
+            return true if progress[:completed]
+
+            completed = false
+            nested_transaction("hecks_backfill_chunk") do
+              @db.exec_params("SELECT pg_advisory_xact_lock(hashtext('hecks_field_cache:' || $1))", [target])
+              progress = backfill_progress(target)
+              if progress[:completed]
+                completed = true
+                next
+              end
+
+              rows = @db.exec(source_sql.call(progress[:cursor]))
+              if rows.ntuples.zero?
+                upsert_backfill_progress!(target, cursor: progress[:cursor], completed: true)
+                completed = true
+                next
+              end
+
+              upsert.call(rows)
+              completed = rows.ntuples < CHUNK_SIZE
+              # `PG::Result#[]` supports neither an out-of-range index NOR
+              # a negative one (unlike a plain Ruby Array) — same trap as
+              # `backfill_progress` above, the explicit last-index form.
+              last_cursor = rows[rows.ntuples - 1]["id"]
+              upsert_backfill_progress!(target, cursor: last_cursor, completed: completed)
+            end
+            completed
+          end
+
+          # `PG::Result#[]` RAISES IndexError on an out-of-range index —
+          # unlike a plain Ruby Array, it does not return nil — so the
+          # ENTIRELY ORDINARY case of "no progress row exists yet" (every
+          # target's very first check) cannot be read via a bare `[0]`.
+          # `ntuples.zero?` first, always.
+          def backfill_progress(target)
+            result = @db.exec_params(
+              "SELECT cursor, completed FROM hecks_backfill_progress WHERE target = $1", [target]
+            )
+            return { cursor: nil, completed: false } if result.ntuples.zero?
+
+            row = result[0]
+            { cursor: row["cursor"], completed: row["completed"] == "t" }
+          end
+
+          def upsert_backfill_progress!(target, cursor:, completed:)
+            @db.exec_params(
+              "INSERT INTO hecks_backfill_progress (target, cursor, completed, updated_at) " \
+              "VALUES ($1, $2, $3, now()) " \
+              "ON CONFLICT (target) DO UPDATE SET cursor = EXCLUDED.cursor, " \
+              "completed = EXCLUDED.completed, updated_at = EXCLUDED.updated_at",
+              [target, cursor, completed]
+            )
+          end
+        end
+      end
+    end
+  end
+end