@intentius/chant-lexicon-sql 0.124.31 → 0.124.32

package/dist/postgres/live/client.js

@@ -0,0 +1,151 @@
+/**
+ * The Postgres wire client: node-postgres (`pg`), loaded the first time a
+ * Postgres server is read, so a ClickHouse-only project never loads it.
+ *
+ * Every session reads the catalog with an empty `search_path`, as `pg_dump`
+ * does: `pg_get_*def()` and `format_type()` then qualify every name outside
+ * `pg_catalog`, so what a server prints does not depend on the role's own
+ * search path.
+ */
+/** A failed statement, with the server's SQLSTATE when it sent one. */
+export class PostgresQueryError extends Error {
+    constructor(message, code, position) {
+        super(message);
+        this.code = code;
+        this.position = position;
+        this.name = "PostgresQueryError";
+    }
+}
+let loaded;
+async function pgModule() {
+    loaded ?? (loaded = import("pg").then((m) => (m.default ?? m)));
+    return loaded;
+}
+const wrap = (err) => {
+    const e = err;
+    return new PostgresQueryError(e.message ?? String(err), e.code, e.position !== undefined ? Number(e.position) : undefined);
+};
+/**
+ * The URL with libpq's meaning of `sslmode`. pg-connection-string reads
+ * `sslmode=require` as verify-full unless `uselibpqcompat=true` is set, so an
+ * RDS host without its CA bundle, or a tunnel whose host name doesn't match
+ * the certificate, fails to connect. In psql, `require` encrypts without
+ * verifying, and a profile url means what it means there. A url that sets
+ * `uselibpqcompat` itself is left alone.
+ */
+export function withLibpqSslMode(url) {
+    if (url.searchParams.has("sslmode") && !url.searchParams.has("uselibpqcompat"))
+        url.searchParams.set("uselibpqcompat", "true");
+    return url;
+}
+/** Connect to `endpoint` and set the catalog-reading session up. */
+export async function connectPostgres(endpoint, options = {}) {
+    const pg = await pgModule();
+    // node-postgres lets the connection string's fields win over the config's, so the credentials go into the URL.
+    const url = withLibpqSslMode(new URL(endpoint.url));
+    if (endpoint.user !== undefined)
+        url.username = encodeURIComponent(endpoint.user);
+    const password = endpoint.token ? await endpoint.token.get() : endpoint.password;
+    if (password !== undefined)
+        url.password = encodeURIComponent(password);
+    const client = new pg.Client({ connectionString: url.toString(), application_name: options.applicationName ?? "chant" });
+    // A server that goes away emits an error event; the next query fails with it instead of the process crashing.
+    client.on("error", () => undefined);
+    try {
+        await client.connect();
+        await client.query("SELECT pg_catalog.set_config('search_path', '', false)");
+    }
+    catch (err) {
+        await client.end().catch(() => undefined);
+        const wrapped = wrap(err);
+        // A refused token is not reused: the next connect mints another.
+        if (endpoint.token && (wrapped.code === "28P01" || wrapped.code === "28000"))
+            endpoint.token.invalidate();
+        throw wrapped;
+    }
+    let ended = false;
+    return {
+        async query(sql, params) {
+            try {
+                const r = await client.query(sql, params);
+                return r.rows;
+            }
+            catch (err) {
+                throw wrap(err);
+            }
+        },
+        async end() {
+            if (ended)
+                return;
+            ended = true;
+            await client.end().catch(() => undefined);
+        },
+    };
+}
+/** SQLSTATE lock_not_available: `lock_timeout` passed while a statement waited for a lock. */
+export const SQLSTATE_LOCK_NOT_AVAILABLE = "55P03";
+/** The `lock_timeout` a statement waits under when the profile sets none (`sql.profiles.<env>.lockTimeoutMs`): the applier's and the catalog reader's. */
+export const DEFAULT_LOCK_TIMEOUT_MS = 5000;
+/**
+ * The sessions holding ACCESS EXCLUSIVE on a relation in the current database,
+ * other than this one. Only that mode conflicts with the ACCESS SHARE a
+ * catalog function (`pg_get_viewdef()`, `pg_get_indexdef()`) takes. Empty when
+ * the lookup itself fails.
+ */
+export async function accessExclusiveHolders(client) {
+    try {
+        const rows = await client.query(`SELECT l.pid, l.relation::pg_catalog.regclass::text AS relation, a.application_name AS application, a.state,
+              (EXTRACT(EPOCH FROM pg_catalog.now() - a.xact_start))::int AS seconds, pg_catalog.left(a.query, 120) AS query
+         FROM pg_catalog.pg_locks l LEFT JOIN pg_catalog.pg_stat_activity a ON a.pid = l.pid
+        WHERE l.locktype = 'relation' AND l.granted AND l.mode = 'AccessExclusiveLock'
+          AND l.pid <> pg_catalog.pg_backend_pid()
+          AND l.database = (SELECT oid FROM pg_catalog.pg_database WHERE datname = pg_catalog.current_database())
+        ORDER BY l.pid, 2 LIMIT 10`);
+        return rows.map((r) => ({
+            pid: Number(r.pid),
+            relation: r.relation,
+            ...(r.application ? { application: r.application } : {}),
+            ...(r.state ? { state: r.state } : {}),
+            ...(r.seconds !== null && r.seconds !== undefined ? { seconds: Number(r.seconds) } : {}),
+            ...(r.query ? { query: r.query.replace(/\s+/g, " ").trim() } : {}),
+        }));
+    }
+    catch {
+        return [];
+    }
+}
+/** One line naming the lock holders, for a message. */
+export function describeLockHolders(holders) {
+    if (holders.length === 0)
+        return "no session holds ACCESS EXCLUSIVE now; the lock was released after the wait";
+    return holders
+        .map((h) => {
+        const about = [h.application ? `application ${JSON.stringify(h.application)}` : "", h.state ?? "", h.seconds !== undefined ? `transaction open ${h.seconds}s` : "", h.query ? `last statement: ${h.query}` : ""].filter(Boolean);
+        return `${h.relation} is held in ACCESS EXCLUSIVE by pid ${h.pid}${about.length > 0 ? ` (${about.join(", ")})` : ""}`;
+    })
+        .join("; ");
+}
+/**
+ * A client for catalog reads: the session's `lock_timeout` is `lockTimeoutMs`
+ * (0 waits without limit, as Postgres does), so a read that needs a table
+ * another session holds in ACCESS EXCLUSIVE (a long `ALTER TABLE`, `LOCK
+ * TABLE`) fails after the wait instead of hanging. The failure names the
+ * relation and the pid holding it.
+ */
+export async function withCatalogLockTimeout(client, lockTimeoutMs = DEFAULT_LOCK_TIMEOUT_MS) {
+    await client.query("SELECT pg_catalog.set_config('lock_timeout', $1, false)", [`${lockTimeoutMs}ms`]);
+    return {
+        async query(sql, params) {
+            try {
+                return await client.query(sql, params);
+            }
+            catch (err) {
+                if (!(err instanceof PostgresQueryError) || err.code !== SQLSTATE_LOCK_NOT_AVAILABLE)
+                    throw err;
+                const holders = describeLockHolders(await accessExclusiveHolders(client));
+                throw new PostgresQueryError(`${err.message.split("\n")[0]}: the catalog read waited ${lockTimeoutMs}ms for a lock; ${holders}. Wait for that transaction to finish, or raise sql.profiles.<env>.lockTimeoutMs`, err.code, err.position);
+            }
+        },
+        end: () => client.end(),
+    };
+}

package/dist/postgres/live/describe-resources.js

@@ -0,0 +1,135 @@
+/**
+ * `describeResources()` for the Postgres dialect: which declared objects exist
+ * on the environment's server.
+ *
+ * One catalog read per run (`./catalog.ts`), then a lookup per declared
+ * entity, so N entities are a fixed handful of queries. An object the catalog
+ * does not list is absent, and the `queried` address names the server and the
+ * object asked for. A server that cannot be reached, or refuses the
+ * credentials, leaves every entity unobserved with that reason: never absent.
+ *
+ * Ownership is chant's marker, the trailer on the object's comment
+ * (`../../core/ownership.ts`): `owned` when the comment carries it, `foreign`
+ * when it does not. With `owned: true` a foreign object is withheld as
+ * `filtered`. An object another tool keeps (an ORM's revision table) is
+ * foreign whatever its comment says.
+ */
+import { observeEntities, } from "@intentius/chant/observation";
+import { bindPostgres, classifyPostgresFailure, redactUrl } from "./bind.js";
+import { liveKey, markProviderOwned, readLiveSchema } from "./catalog.js";
+import { POSTGRES_ENTITY_TYPES } from "../entity-types.js";
+import { isChantManaged, readMarker, stripMarker } from "../../core/ownership.js";
+import { identValue } from "../parser.js";
+import { canonicalPgObject } from "../plan/normalize.js";
+/**
+ * The schema and name a declared entity is created as; a bare name is in the
+ * default schema. A routine's parameter types and a trigger's table tell it
+ * from another of the same name (`signature`).
+ */
+export function declaredAddress(entity, defaultSchema) {
+    const name = String(entity.props.name ?? "");
+    if (entity.type === POSTGRES_ENTITY_TYPES.schema || entity.type === POSTGRES_ENTITY_TYPES.extension || entity.type === POSTGRES_ENTITY_TYPES.role || entity.type === POSTGRES_ENTITY_TYPES.eventTrigger)
+        return { name };
+    // A cast is named by its two types, as the catalog prints them.
+    if (entity.type === POSTGRES_ENTITY_TYPES.cast)
+        return { name: canonicalPgObject(entity.type, entity.props, defaultSchema).name };
+    const schema = typeof entity.props.schema === "string" ? entity.props.schema : identValue(defaultSchema);
+    if (entity.type === POSTGRES_ENTITY_TYPES.function ||
+        entity.type === POSTGRES_ENTITY_TYPES.procedure ||
+        entity.type === POSTGRES_ENTITY_TYPES.aggregate ||
+        entity.type === POSTGRES_ENTITY_TYPES.trigger ||
+        entity.type === POSTGRES_ENTITY_TYPES.policy) {
+        const signature = canonicalPgObject(entity.type, entity.props, defaultSchema).signature;
+        return { schema, name, ...(signature !== undefined ? { signature } : {}) };
+    }
+    return { schema, name };
+}
+/**
+ * Why an access declaration is not observed one by one: a grant or default
+ * privileges, which `chant sql plan` compares as the access they add up to;
+ * a policy or role where the profile does not manage access.
+ */
+export function accessUnobserved(type, managed) {
+    if (type === POSTGRES_ENTITY_TYPES.grant || type === POSTGRES_ENTITY_TYPES.defaultPrivileges) {
+        return { reason: "unsupported-kind", detail: "a grant is compared as the privileges the declarations add up to, by chant sql plan and the applier, not one statement at a time" };
+    }
+    if (!managed && (type === POSTGRES_ENTITY_TYPES.policy || type === POSTGRES_ENTITY_TYPES.role)) {
+        return { reason: "filtered", detail: "access is not managed in this environment (sql.profiles.<env>.access)" };
+    }
+    return undefined;
+}
+/** A schema scope for the catalog read: the profile's schemas, else every schema the declarations name and the default one. */
+export function scopeFor(target, entities) {
+    if (target.schemas)
+        return target.schemas;
+    if (entities.length === 0)
+        return undefined;
+    const out = new Set([target.defaultSchema]);
+    // The schema alone: the rest of an address (a routine's parameter types) needs props a caller may not have.
+    const unplaced = new Set([POSTGRES_ENTITY_TYPES.extension, POSTGRES_ENTITY_TYPES.role, POSTGRES_ENTITY_TYPES.cast, POSTGRES_ENTITY_TYPES.eventTrigger]);
+    for (const e of entities) {
+        if (e.type === POSTGRES_ENTITY_TYPES.schema)
+            out.add(String(e.props.name ?? ""));
+        else if (!unplaced.has(e.type))
+            out.add(typeof e.props.schema === "string" ? e.props.schema : identValue(target.defaultSchema));
+    }
+    return [...out].sort();
+}
+function adapter(options) {
+    return {
+        async bind() {
+            const { target, client } = await bindPostgres(options);
+            try {
+                const live = markProviderOwned(await readLiveSchema(client, {
+                    schemas: scopeFor(target, options.declared.filter((d) => d.type.startsWith("Postgres::"))),
+                    access: target.access === true,
+                    roles: options.declared.filter((d) => d.type === POSTGRES_ENTITY_TYPES.role).map((d) => String(d.props.name)),
+                }), target.provider);
+                return { target, byKey: new Map(live.map((o) => [liveKey(o.type, o.schema, o.name, o.signature), o])) };
+            }
+            finally {
+                await client.end();
+            }
+        },
+        classifyBindFailure: (err) => classifyPostgresFailure(err),
+        async read({ target, byKey }, entity) {
+            if (!entity.type.startsWith("Postgres::"))
+                return { unobserved: { reason: "unsupported-kind", detail: entity.type } };
+            const why = accessUnobserved(entity.type, target.access === true);
+            if (why)
+                return { unobserved: why };
+            const { schema, name, signature } = declaredAddress(entity, target.defaultSchema);
+            const queried = `${redactUrl(target.endpoint.url)} ${schema ? `${schema}.` : ""}${name}${signature ?? ""}`;
+            const live = byKey.get(liveKey(entity.type, schema, name, signature));
+            if (!live)
+                return { absent: true, queried };
+            const owned = !live.foreign && isChantManaged(live.comment);
+            if (options.owned && !owned) {
+                return {
+                    unobserved: { reason: "filtered", detail: live.foreign ? `${live.foreign} keeps this object` : "the object's comment carries no chant ownership marker and owned was requested" },
+                    queried,
+                };
+            }
+            const marker = readMarker(live.comment);
+            const comment = live.comment ? stripMarker(live.comment) : "";
+            return {
+                present: {
+                    type: live.type,
+                    physicalId: live.oid,
+                    status: "PRESENT",
+                    attributes: { oid: live.oid, ...(comment ? { comment } : {}), ...(live.foreign ? { keptBy: live.foreign } : {}) },
+                    ownership: owned ? "owned" : "foreign",
+                    ...(owned && marker?.stack ? { marker: { stack: marker.stack, ...(marker.env ? { env: marker.env } : {}) } } : {}),
+                },
+                queried,
+            };
+        },
+    };
+}
+export async function describeResources(options) {
+    const declared = options.entityNames.map((name) => {
+        const entity = options.entities.get(name);
+        return { name, type: entity?.entityType ?? "", props: entity?.props ?? {} };
+    });
+    return observeEntities(declared, adapter({ ...options, declared }));
+}

package/dist/postgres/live/undeclarable.js

@@ -0,0 +1,111 @@
+/**
+ * The objects in a live Postgres database of the kinds no declaration holds
+ * (#3911), found so import can refuse them by name instead of leaving them
+ * out (`../../core/undeclarable.ts`).
+ *
+ * One catalog query per kind the coverage table marks refused
+ * (`../../coverage/postgres.json`), keyed by the row's `kind`. A kind whose
+ * row is flipped to declared is no longer asked about: its objects are then
+ * the catalog reader's (`./catalog.ts`) to read.
+ *
+ * Left out, as the reader leaves them out: the server's own schemas, objects
+ * an extension owns (`pg_depend.deptype = 'e'`; the extension is declared),
+ * the objects `initdb` makes, whose oids are below 16384
+ * (`FirstNormalObjectId`), and an object that is part of another
+ * (`deptype = 'i'`: the casts a range type makes for itself). An object in a
+ * schema is looked for in the schemas in scope; one that belongs to the whole
+ * database (a cast, an event trigger, a publication, a foreign data wrapper)
+ * is always looked for.
+ */
+import { notExtensionOwned, scopeSql } from "./catalog.js";
+import { KIND_COVERAGE } from "../../coverage.js";
+/** `FirstNormalObjectId`: every oid below it is the server's own. */
+const FIRST_USER_OID = 16384;
+const C = 'COLLATE "C"';
+/** Not part of another object (`pg_depend.deptype = 'i'`): the multirange cast a range type makes for itself is the range type's. */
+const notInternal = (catalog, oid) => `NOT EXISTS (SELECT 1 FROM pg_catalog.pg_depend d WHERE d.classid = 'pg_catalog.${catalog}'::pg_catalog.regclass AND d.objid = ${oid} AND d.deptype = 'i')`;
+const ns = (alias, column) => `JOIN pg_catalog.pg_namespace ${alias} ON ${alias}.oid = ${column}`;
+const args = (oid) => `'(' || pg_catalog.replace(pg_catalog.oidvectortypes(${oid}), ', ', ',') || ')'`;
+/** A probe for objects of a catalog in a schema, the catalog aliased `x`; `name` is the name expression. */
+const inSchema = (catalog, nsColumn, name, where = "TRUE") => ({
+    sql: (scope) => `SELECT n.nspname AS schema, ${name} AS name, NULL::text AS owner FROM pg_catalog.${catalog} x ${ns("n", `x.${nsColumn}`)}
+     WHERE ${where} AND ${scope} AND x.oid >= ${FIRST_USER_OID} AND ${notExtensionOwned(catalog, "x.oid")} AND ${notInternal(catalog, "x.oid")}`,
+});
+/** A probe for objects that belong to the database, not a schema. */
+const databaseWide = (catalog, name, owner) => ({
+    sql: () => `SELECT NULL::text AS schema, ${name} AS name, ${owner ? `pg_catalog.pg_get_userbyid(x.${owner})` : "NULL::text"} AS owner FROM pg_catalog.${catalog} x
+     WHERE x.oid >= ${FIRST_USER_OID} AND ${notExtensionOwned(catalog, "x.oid")} AND ${notInternal(catalog, "x.oid")}`,
+});
+/** The pg_type rows of one `typtype`. */
+const type = (where) => inSchema("pg_type", "typnamespace", "x.typname", where);
+export const POSTGRES_PROBES = {
+    "access method": databaseWide("pg_am", "x.amname"),
+    aggregate: inSchema("pg_proc", "pronamespace", `x.proname || ${args("x.proargtypes")}`, "x.prokind = 'a'"),
+    "routine with a SQL-standard body": inSchema("pg_proc", "pronamespace", `x.proname || ${args("x.proargtypes")}`, "x.prokind IN ('f', 'p', 'w') AND x.prosqlbody IS NOT NULL"),
+    cast: databaseWide("pg_cast", "'(' || pg_catalog.format_type(x.castsource, NULL) || ' AS ' || pg_catalog.format_type(x.casttarget, NULL) || ')'"),
+    collation: inSchema("pg_collation", "collnamespace", "x.collname"),
+    conversion: inSchema("pg_conversion", "connamespace", "x.conname"),
+    "event trigger": databaseWide("pg_event_trigger", "x.evtname", "evtowner"),
+    "foreign data wrapper": databaseWide("pg_foreign_data_wrapper", "x.fdwname", "fdwowner"),
+    "foreign server": databaseWide("pg_foreign_server", "x.srvname", "srvowner"),
+    "user mapping": {
+        // pg_user_mapping is the superuser's to read; the view shows every mapping without its options.
+        sql: () => `SELECT NULL::text AS schema, 'for ' || x.usename || ' on server ' || x.srvname AS name, pg_catalog.pg_get_userbyid(s.srvowner) AS owner FROM pg_catalog.pg_user_mappings x
+       JOIN pg_catalog.pg_foreign_server s ON s.oid = x.srvid
+       WHERE ${notExtensionOwned("pg_foreign_server", "s.oid")}`,
+    },
+    "foreign table": inSchema("pg_class", "relnamespace", "x.relname", "x.relkind = 'f'"),
+    "procedural language": databaseWide("pg_language", "x.lanname", "lanowner"),
+    operator: inSchema("pg_operator", "oprnamespace", "x.oprname || '(' || pg_catalog.format_type(x.oprleft, NULL) || ',' || pg_catalog.format_type(x.oprright, NULL) || ')'"),
+    "operator class": inSchema("pg_opclass", "opcnamespace", "x.opcname || ' USING ' || (SELECT a.amname FROM pg_catalog.pg_am a WHERE a.oid = x.opcmethod)"),
+    // A family CREATE OPERATOR CLASS made for itself, named as its class, is the class's.
+    "operator family": inSchema("pg_opfamily", "opfnamespace", "x.opfname || ' USING ' || (SELECT a.amname FROM pg_catalog.pg_am a WHERE a.oid = x.opfmethod)", "NOT EXISTS (SELECT 1 FROM pg_catalog.pg_opclass c WHERE c.opcfamily = x.oid AND c.opcname = x.opfname AND c.opcnamespace = x.opfnamespace)"),
+    publication: databaseWide("pg_publication", "x.pubname", "pubowner"),
+    subscription: {
+        sql: () => `SELECT NULL::text AS schema, x.subname AS name, pg_catalog.pg_get_userbyid(x.subowner) AS owner FROM pg_catalog.pg_subscription x
+       WHERE x.subdbid = (SELECT d.oid FROM pg_catalog.pg_database d WHERE d.datname = pg_catalog.current_database())`,
+    },
+    rule: {
+        sql: (scope) => `SELECT n.nspname AS schema, r.rulename || ' ON ' || c.relname AS name, NULL::text AS owner
+       FROM pg_catalog.pg_rewrite r JOIN pg_catalog.pg_class c ON c.oid = r.ev_class ${ns("n", "c.relnamespace")}
+       WHERE r.rulename <> '_RETURN' AND ${scope} AND ${notExtensionOwned("pg_class", "c.oid")}`,
+    },
+    "extended statistics": inSchema("pg_statistic_ext", "stxnamespace", "x.stxname"),
+    "text search configuration": inSchema("pg_ts_config", "cfgnamespace", "x.cfgname"),
+    "text search dictionary": inSchema("pg_ts_dict", "dictnamespace", "x.dictname"),
+    "text search parser": inSchema("pg_ts_parser", "prsnamespace", "x.prsname"),
+    "text search template": inSchema("pg_ts_template", "tmplnamespace", "x.tmplname"),
+    transform: databaseWide("pg_transform", "'FOR ' || pg_catalog.format_type(x.trftype, NULL) || ' LANGUAGE ' || (SELECT l.lanname FROM pg_catalog.pg_language l WHERE l.oid = x.trflang)"),
+    "composite type": type("x.typtype = 'c' AND EXISTS (SELECT 1 FROM pg_catalog.pg_class c WHERE c.oid = x.typrelid AND c.relkind = 'c')"),
+    "range type": type("x.typtype = 'r'"),
+    // A base type, or a shell; not the array type the server makes beside every type.
+    "base type": type("x.typtype IN ('b', 'p') AND NOT EXISTS (SELECT 1 FROM pg_catalog.pg_type e WHERE e.typarray = x.oid)"),
+};
+/**
+ * Every object in scope of a kind the coverage table refuses, kind by kind
+ * in the table's order, each by schema then name. A kind the session may not
+ * read (a role without the privilege) is a problem in `unread`, not a pass.
+ */
+export async function readUndeclarable(client, scope = {}) {
+    const s = scopeSql("n.nspname", scope);
+    const objects = [];
+    const unread = [];
+    for (const row of KIND_COVERAGE.postgres) {
+        if (row.status !== "refused")
+            continue;
+        const probe = POSTGRES_PROBES[row.kind];
+        if (!probe)
+            throw new Error(`no catalog query finds a Postgres ${row.kind} (src/postgres/live/undeclarable.ts)`);
+        // The comment names the kind, in the server's log and to a test's fake client.
+        const sql = `/* chant undeclarable: ${row.kind} */ SELECT q.schema, q.name, q.owner FROM (${probe.sql(s.sql)}) q ORDER BY q.schema ${C} NULLS FIRST, q.name ${C}`;
+        try {
+            const rows = await client.query(sql, sql.includes("$1") ? s.params : []);
+            for (const r of rows)
+                objects.push({ kind: row.kind, ...(r.schema ? { schema: r.schema } : {}), name: r.name, ...(r.owner ? { owner: r.owner } : {}) });
+        }
+        catch (err) {
+            unread.push(`${row.kind}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    return { objects, unread };
+}

package/dist/postgres/migrate/backfill.js

@@ -0,0 +1,170 @@
+/**
+ * The backfill and the verification of the expand-and-contract migration
+ * (#3281).
+ *
+ * ## Backfill
+ *
+ * The rows that were in the table before the dual write started are filled
+ * in batches: one range of the primary key per batch (`./batches.ts`:
+ * `[b * size, (b + 1) * size)` of an integer key, else between boundaries
+ * recorded in the receipt store), so a batch is the same rows on every run
+ * however many rows were written since. Each batch is one transaction:
+ *
+ *     UPDATE t SET new = <expression> WHERE key >= lo AND key < hi AND new IS DISTINCT FROM <expression>;
+ *     INSERT INTO __chant_receipts ... -- the batch's receipt, last
+ *     COMMIT;
+ *
+ * under `lock_timeout` (a row an application transaction holds makes the
+ * batch give up and try again rather than queue behind it) and the
+ * statement timeout for a catalog change, which bounds how long one batch
+ * may hold its row locks.
+ *
+ * Per batch, the cycle core's `effect()` step runs
+ * (concepts/effect-receipts.mdx): read the receipt, skip the batch when it
+ * matches, otherwise update and write the receipt last, on success only.
+ * The batches are only known when the run reads the table, so the cycle runs
+ * here over the shared core's `ReceiptStore` seam (`./receipts.ts`), as the
+ * ClickHouse rebuild's does per partition. Unlike ClickHouse, the receipt
+ * commits in the batch's own transaction: a run killed anywhere leaves a
+ * batch with its receipt or neither, and a resumed backfill starts at the
+ * first batch with no receipt and updates no batch twice.
+ *
+ * Before each batch the backfill waits while a replica is further behind
+ * than the bound (`./replication.ts`).
+ *
+ * ## Verification
+ *
+ * One statement over the whole table, one snapshot: the rows, the rows whose
+ * new column differs from the expression over the old one, the new column's
+ * NULLs, and a checksum (the sum of a 64-bit hash of each value) of the new
+ * column and of the expression. Any difference, or a NULL where the column
+ * is declared NOT NULL, fails the run, whose onFailure then drops what the
+ * expand added. The dual-write trigger keeps writes going during the
+ * verification consistent; the statement reads the table under ACCESS SHARE
+ * only.
+ */
+import { EffectReceipt, receiptExpectation } from "@intentius/chant/effect-receipt";
+import { computePlanDigest } from "@intentius/chant/op";
+import { col } from "./names.js";
+import { MigrationRefusal, migrationPlanSubject } from "./observe.js";
+import { postgresReceiptStore, receiptAddress } from "./receipts.js";
+import { waitForReplicas } from "./replication.js";
+import { batchPrefix, identityOf, inTransaction, observe } from "./steps.js";
+import { carriedNotReady, requiresNotNull } from "./carry.js";
+import { integerBatches, isIntegerKey, recordedBatches } from "./batches.js";
+/** The effect a batch is, and its receipt's address suffix. */
+export const batchEffect = (key, batch) => `migrate/${key}/${batch}`;
+/** The receipt's expected value for one batch: bound to the table, the new column's attribute number, the batch's rows and the expression. */
+export function batchExpectation(o, batch) {
+    const effect = batchEffect(o.names.key, batch.id);
+    return receiptExpectation(EffectReceipt(effect, {
+        effect,
+        flavor: "hash",
+        inputs: { migration: o.names.key, table: o.oid, column: o.newColumn.attnum, ...batch.inputs, expression: o.expression },
+    }));
+}
+/** The address suffix of the recorded batch boundaries of a key that is not one integer column. */
+export const BATCHES_RECORD = "batches";
+export async function backfill(run, deps = {}) {
+    const o = await observe(run);
+    const result = { state: o.state, batches: 0, filled: 0, skipped: 0, rows: 0, pausedMs: 0 };
+    if (o.state !== "migrate")
+        return result;
+    const n = o.names;
+    if (!o.newColumn)
+        throw new MigrationRefusal(`${n.key}: the new column is not there yet; the Expand phase adds it`);
+    if (!o.trigger)
+        throw new MigrationRefusal(`${n.key}: the dual write is not on; the Dual write phase starts it, and the backfill fills only rows written before it`);
+    if (!Number.isInteger(run.batchSize) || run.batchSize < 1)
+        throw new MigrationRefusal(`${n.key}: batchSize must be a positive integer, got ${run.batchSize}`);
+    const identity = identityOf(run.marker);
+    const receipts = deps.receipts ?? postgresReceiptStore(run.client, n.schema, identity, run.runId ? { runId: run.runId } : {});
+    await receipts.ensure();
+    const recorded = await receipts.readAll(batchPrefix(identity, n.key));
+    const batches = isIntegerKey(o.keyColumns)
+        ? await integerBatches(run.client, n.qualifiedTable, o.keyColumns[0], run.batchSize)
+        : (await recordedBatches(run.client, {
+            table: n.qualifiedTable,
+            oid: o.oid,
+            attnum: o.newColumn.attnum,
+            keys: o.keyColumns,
+            size: run.batchSize,
+            effect: batchEffect(n.key, BATCHES_RECORD),
+            receipts,
+            log: run.log,
+            ...(run.signal ? { signal: run.signal } : {}),
+        })).batches;
+    result.batches = batches.length;
+    const wait = deps.waitForReplicas ?? waitForReplicas;
+    const target = `${col(n.newColumn)}`;
+    for (const b of batches) {
+        run.signal?.throwIfAborted();
+        const effect = batchEffect(n.key, b.id);
+        const expectation = batchExpectation(o, b);
+        if (recorded.get(receiptAddress(identity, effect)) === expectation) {
+            result.skipped++;
+            continue;
+        }
+        if (run.replicationLag)
+            result.pausedMs += await wait(run.client, run.replicationLag, { log: run.log, ...(run.signal ? { signal: run.signal } : {}) });
+        const updated = await inTransaction(run, async (exec) => {
+            const [row] = await exec(`WITH u AS (UPDATE ${n.qualifiedTable} SET ${target} = (${o.expression}) WHERE ${b.where} AND ${target} IS DISTINCT FROM (${o.expression}) RETURNING 1) ` +
+                "SELECT count(*)::int AS n FROM u", b.params);
+            // The receipt, last, in the batch's own transaction.
+            await receipts.write({ name: effect, effect, flavor: "hash", inputs: {} }, expectation);
+            await deps.beforeCommit?.(b.id);
+            return Number(row?.n ?? 0);
+        });
+        result.rows += updated;
+        result.filled++;
+        await deps.afterBatch?.(b.id);
+    }
+    run.log(`-- backfill of ${n.key}: ${result.batches} batch(es), ${result.filled} filled (${result.rows} row(s)), ${result.skipped} already filled`);
+    return result;
+}
+/** A verification that found the columns different. */
+export class MigrationVerificationError extends Error {
+    constructor(key, result, reason) {
+        super(`${key}: ${reason}. Nothing was switched; onFailure drops what the expand added, and the next run starts again.`);
+        this.key = key;
+        this.result = result;
+        this.name = "MigrationVerificationError";
+    }
+}
+export async function verifyMigration(run) {
+    const o = await observe(run);
+    const empty = { rows: 0, mismatched: 0, nulls: 0, checksum: "0", expected: "0" };
+    if (o.state !== "migrate")
+        return { state: o.state, ...empty, summary: `nothing to verify: ${o.state}` };
+    const n = o.names;
+    if (!o.newColumn || !o.trigger)
+        throw new MigrationRefusal(`${n.key}: the new column and its dual write are not there; the Expand and Dual write phases make them`);
+    const nw = col(n.newColumn);
+    const [row] = await inTransaction(run, (exec) => exec(`SELECT count(*)::text AS rows, count(*) FILTER (WHERE ${nw} IS DISTINCT FROM (${o.expression}))::text AS mismatched,
+                count(*) FILTER (WHERE ${nw} IS NULL)::text AS nulls,
+                COALESCE(sum(pg_catalog.hashtextextended(${nw}::text, 0)), 0)::text AS checksum,
+                COALESCE(sum(pg_catalog.hashtextextended((${o.expression})::text, 0)), 0)::text AS expected
+         FROM ${n.qualifiedTable}`), { scan: true });
+    const r = { rows: Number(row?.rows ?? 0), mismatched: Number(row?.mismatched ?? 0), nulls: Number(row?.nulls ?? 0), checksum: String(row?.checksum ?? "0"), expected: String(row?.expected ?? "0") };
+    if (r.mismatched > 0 || r.checksum !== r.expected) {
+        throw new MigrationVerificationError(n.key, { state: o.state, ...r }, `${r.mismatched} of ${r.rows} row(s) have ${n.newColumn} different from ${o.expression} (checksums ${r.checksum} and ${r.expected})`);
+    }
+    if (requiresNotNull(o.column, o.carried) && r.nulls > 0) {
+        throw new MigrationVerificationError(n.key, { state: o.state, ...r }, `${r.nulls} of ${r.rows} row(s) have ${n.newColumn} NULL, and ${n.column} is declared NOT NULL or is in the primary key`);
+    }
+    const notReady = carriedNotReady(o.carried, o.carriedStates);
+    if (notReady.length > 0) {
+        throw new MigrationRefusal(`${n.key}: ${notReady.map((c) => `${c.kind} ${c.working}`).join(", ")} on the new column is not there or not yet valid; the Carry over phase makes them`);
+    }
+    const carried = o.carried.length > 0 || o.views.length > 0
+        ? `; ${o.carried.map((c) => `${c.kind} ${c.name}${c.target !== c.name ? ` (as ${c.target})` : ""}`).join(", ") || "no index or constraint"} carried over${o.views.length > 0 ? `, view(s) ${o.views.map((v) => v.name).join(", ")} made again at the switch` : ""}`
+        : "";
+    const summary = `${n.key}: ${r.rows} row(s), ${n.newColumn} equal to ${o.expression} in every one (checksum ${r.checksum})${carried}`;
+    run.log(`-- ${summary}`);
+    return {
+        state: o.state,
+        planDigest: computePlanDigest("postgres-migration", { plan: migrationPlanSubject(o), table: o.oid, column: o.newColumn.attnum, verified: { mismatched: 0 } }),
+        ...r,
+        summary,
+    };
+}