@intentius/chant-lexicon-sql 0.0.0-stage → 0.112.1

package/src/postgres/live/describe-resources.ts

@@ -0,0 +1,111 @@
+/**
+ * `describeResources()` for the Postgres dialect: which declared objects exist
+ * on the environment's server.
+ *
+ * One catalog read per run (`./catalog.ts`), then a lookup per declared
+ * entity, so N entities are a fixed handful of queries. An object the catalog
+ * does not list is absent, and the `queried` address names the server and the
+ * object asked for. A server that cannot be reached, or refuses the
+ * credentials, leaves every entity unobserved with that reason: never absent.
+ *
+ * Ownership is chant's marker, the trailer on the object's comment
+ * (`../../core/ownership.ts`): `owned` when the comment carries it, `foreign`
+ * when it does not. With `owned: true` a foreign object is withheld as
+ * `filtered`. An object another tool keeps (an ORM's revision table) is
+ * foreign whatever its comment says.
+ */
+
+import {
+  observeEntities,
+  type DeclaredEntity,
+  type DescribeResourcesResult,
+  type EntityObservation,
+  type ObserverAdapter,
+} from "@intentius/chant/observation";
+import { bindPostgres, classifyPostgresFailure, redactUrl, type BindOptions, type PostgresTarget } from "./bind";
+import { liveKey, markProviderOwned, readLiveSchema, type LivePgObject } from "./catalog";
+import { POSTGRES_ENTITY_TYPES } from "../entity-types";
+import { isChantManaged, readMarker, stripMarker } from "../../core/ownership";
+import { identValue } from "../parser";
+
+interface Bound {
+  target: PostgresTarget;
+  byKey: Map<string, LivePgObject>;
+}
+
+/** The schema and name a declared entity is created as; a bare name is in the default schema. */
+export function declaredAddress(entity: { type: string; props: Record<string, unknown> }, defaultSchema: string): { schema?: string; name: string } {
+  const name = String(entity.props.name ?? "");
+  if (entity.type === POSTGRES_ENTITY_TYPES.schema || entity.type === POSTGRES_ENTITY_TYPES.extension) return { name };
+  return { schema: typeof entity.props.schema === "string" ? entity.props.schema : identValue(defaultSchema), name };
+}
+
+/** A schema scope for the catalog read: the profile's schemas, else every schema the declarations name and the default one. */
+export function scopeFor(target: PostgresTarget, entities: ReadonlyArray<{ type: string; props: Record<string, unknown> }>): string[] | undefined {
+  if (target.schemas) return target.schemas;
+  if (entities.length === 0) return undefined;
+  const out = new Set<string>([target.defaultSchema]);
+  for (const e of entities) {
+    const a = declaredAddress(e, target.defaultSchema);
+    if (e.type === POSTGRES_ENTITY_TYPES.schema) out.add(a.name);
+    else if (a.schema) out.add(a.schema);
+  }
+  return [...out].sort();
+}
+
+function adapter(options: BindOptions & { owned?: boolean; declared: DeclaredEntity[] }): ObserverAdapter<Bound> {
+  return {
+    async bind() {
+      const { target, client } = await bindPostgres(options);
+      try {
+        const live = markProviderOwned(await readLiveSchema(client, { schemas: scopeFor(target, options.declared.filter((d) => d.type.startsWith("Postgres::"))) }), target.provider);
+        return { target, byKey: new Map(live.map((o) => [liveKey(o.type, o.schema, o.name), o])) };
+      } finally {
+        await client.end();
+      }
+    },
+    classifyBindFailure: (err) => classifyPostgresFailure(err),
+    async read({ target, byKey }, entity): Promise<EntityObservation> {
+      if (!entity.type.startsWith("Postgres::")) return { unobserved: { reason: "unsupported-kind", detail: entity.type } };
+      const { schema, name } = declaredAddress(entity, target.defaultSchema);
+      const queried = `${redactUrl(target.endpoint.url)} ${schema ? `${schema}.` : ""}${name}`;
+      const live = byKey.get(liveKey(entity.type, schema, name));
+      if (!live) return { absent: true, queried };
+      const owned = !live.foreign && isChantManaged(live.comment);
+      if (options.owned && !owned) {
+        return {
+          unobserved: { reason: "filtered", detail: live.foreign ? `${live.foreign} keeps this object` : "the object's comment carries no chant ownership marker and owned was requested" },
+          queried,
+        };
+      }
+      const marker = readMarker(live.comment);
+      const comment = live.comment ? stripMarker(live.comment) : "";
+      return {
+        present: {
+          type: live.type,
+          physicalId: live.oid,
+          status: "PRESENT",
+          attributes: { oid: live.oid, ...(comment ? { comment } : {}), ...(live.foreign ? { keptBy: live.foreign } : {}) },
+          ownership: owned ? "owned" : "foreign",
+          ...(owned && marker?.stack ? { marker: { stack: marker.stack, ...(marker.env ? { env: marker.env } : {}) } } : {}),
+        },
+        queried,
+      };
+    },
+  };
+}
+
+export async function describeResources(
+  options: {
+    environment: string;
+    entityNames: string[];
+    entities: Map<string, { entityType: string; props: Record<string, unknown> }>;
+    owned?: boolean;
+  } & Omit<BindOptions, "environment">,
+): Promise<DescribeResourcesResult> {
+  const declared: DeclaredEntity[] = options.entityNames.map((name) => {
+    const entity = options.entities.get(name);
+    return { name, type: entity?.entityType ?? "", props: entity?.props ?? {} };
+  });
+  return observeEntities(declared, adapter({ ...options, declared }));
+}

package/src/postgres/live/live.test.ts

@@ -0,0 +1,198 @@
+import { describe, expect, test } from "vitest";
+import { describeObservationConformance } from "@intentius/chant-test-utils";
+import { normalizeObservation } from "@intentius/chant/observation";
+import { classifyPostgresFailure, PostgresBindingError, redactUrl, resolvePostgresTarget } from "./bind";
+import { PostgresQueryError } from "./client";
+import { describeResources } from "./describe-resources";
+import { markProviderOwned, readLiveSchema, sequenceOptions, type LivePgObject } from "./catalog";
+import { fakeClient, type FakeCatalog } from "../testing/fake-client";
+import { dialectOfBinding, dialectOfEntities } from "../../live-dialect";
+import { sqlPlugin } from "../../plugin";
+
+const config = {
+  sql: {
+    profiles: {
+      prod: { url: "postgres://db.internal:5432/shop", user: { env: "PG_USER" }, password: { env: "PG_PASSWORD" }, schemas: ["app"] },
+      analytics: { url: "http://clickhouse:8123" },
+    },
+  },
+};
+
+describe("binding an environment to a Postgres server", () => {
+  test("a postgres:// profile, with its credentials from the environment", () => {
+    expect(resolvePostgresTarget({ environment: "prod", config, env: { PG_USER: "app", PG_PASSWORD: "s3cret" } })).toEqual({
+      endpoint: { url: "postgres://db.internal:5432/shop", user: "app", password: "s3cret" },
+      source: "sql.profiles.prod",
+      schemas: ["app"],
+      defaultSchema: "public",
+    });
+  });
+
+  test("a profile naming an unset credential is no-credentials; nothing bound is no-binding", () => {
+    expect(resolvePostgresTarget({ environment: "prod", config, env: {} })).toMatchObject({ reason: "no-credentials" });
+    expect(resolvePostgresTarget({ environment: "staging", config, env: {} })).toMatchObject({ reason: "no-binding" });
+    expect(resolvePostgresTarget({ environment: "staging", env: { POSTGRES_URL: "postgres://localhost/x" } })).toMatchObject({ source: "env POSTGRES_URL" });
+  });
+
+  test("a refused password is no-credentials, any other failure read-failed, and a URL is shown without its password", () => {
+    expect(classifyPostgresFailure(new PostgresQueryError("password authentication failed", "28P01")).reason).toBe("no-credentials");
+    expect(classifyPostgresFailure(new PostgresQueryError("connection refused")).reason).toBe("read-failed");
+    expect(classifyPostgresFailure(new PostgresBindingError({ reason: "no-binding", detail: "x" })).reason).toBe("no-binding");
+    expect(redactUrl("postgres://app:hunter2@db/shop")).toBe("postgres://app:***@db/shop");
+  });
+
+  test("the dialect a read is for: the declarations' type, else the binding", () => {
+    expect(dialectOfEntities(new Map([["a", { entityType: "Postgres::Table" }]]))).toBe("postgres");
+    expect(dialectOfEntities(new Map([["a", { entityType: "ClickHouse::Table" }]]))).toBe("clickhouse");
+    expect(dialectOfBinding({ environment: "prod", config })).toBe("postgres");
+    expect(dialectOfBinding({ environment: "analytics", config })).toBe("clickhouse");
+    expect(dialectOfBinding({ config: { sql: { dialect: "postgres" } }, env: {} })).toBe("postgres");
+    expect(dialectOfBinding({ env: { POSTGRES_URL: "postgres://x/y" } })).toBe("postgres");
+    expect(dialectOfBinding({ env: {} })).toBe("clickhouse");
+  });
+});
+
+describe("reading the catalog", () => {
+  test("a table prints as the statement that creates it, its comment's trailer taken off", async () => {
+    const [t] = await readLiveSchema(
+      fakeClient({ tables: [{ schema: "app", name: "users", comment: "Accounts [chant managed-by=chant stack=shop]", columns: [{ name: "id", type: "bigint", notnull: true }] }] }),
+    );
+    expect(t).toMatchObject({ type: "Postgres::Table", schema: "app", name: "users", comment: "Accounts [chant managed-by=chant stack=shop]" });
+    expect(t!.statement).toBe("CREATE TABLE app.users (\n    id bigint NOT NULL\n);\nCOMMENT ON TABLE app.users IS 'Accounts'");
+  });
+
+  test("an ORM's revision table is read as that tool's", async () => {
+    const [t] = await readLiveSchema(fakeClient({ tables: [{ schema: "public", name: "_prisma_migrations" }] }));
+    expect(t!.foreign).toBe("Prisma Migrate");
+  });
+
+  test("sequence options at their defaults are left out", () => {
+    const p = { type: "bigint", start: "1", increment: "1", min: "1", max: "9223372036854775807", cache: "1", cycle: false };
+    expect(sequenceOptions(p, true)).toEqual([]);
+    expect(sequenceOptions({ ...p, type: "integer", max: "2147483647", start: "1000" }, true)).toEqual(["AS integer", "START WITH 1000"]);
+    expect(sequenceOptions({ ...p, increment: "-1", min: "-9223372036854775808", max: "-1", start: "-1" }, true)).toEqual(["INCREMENT BY -1"]);
+  });
+});
+
+const CATALOG: FakeCatalog = {
+  schemas: [{ name: "app" }],
+  tables: [
+    { schema: "app", name: "users", comment: "[chant managed-by=chant stack=shop env=prod]" },
+    { schema: "app", name: "legacy" },
+    { schema: "public", name: "schema_migrations" },
+  ],
+};
+const entity = (type: string, props: Record<string, unknown>) => ({ entityType: type, props });
+const options = (names: Record<string, { entityType: string; props: Record<string, unknown> }>, owned = false) => ({
+  environment: "prod",
+  config,
+  env: { PG_USER: "app", PG_PASSWORD: "x" },
+  owned,
+  entityNames: Object.keys(names),
+  entities: new Map(Object.entries(names)),
+  connect: async () => fakeClient(CATALOG),
+});
+const DECLARED = {
+  app: entity("Postgres::Schema", { name: "app" }),
+  users: entity("Postgres::Table", { schema: "app", name: "users" }),
+  legacy: entity("Postgres::Table", { schema: "app", name: "legacy" }),
+  orders: entity("Postgres::Table", { schema: "app", name: "orders" }),
+  migrations: entity("Postgres::Table", { name: "schema_migrations" }),
+};
+
+describe("describeResources", () => {
+  test("present with its ownership, absent with the address asked", async () => {
+    const r = normalizeObservation(await describeResources(options(DECLARED)));
+    expect(r.resources.users).toMatchObject({ ownership: "owned", marker: { stack: "shop", env: "prod" } });
+    expect(r.resources.legacy).toMatchObject({ ownership: "foreign" });
+    expect(r.resources.migrations).toMatchObject({ ownership: "foreign", attributes: { keptBy: expect.stringMatching(/migration runner/) } });
+    expect(r.resources.orders).toBeUndefined();
+    expect(r.queried.orders).toBe("postgres://db.internal:5432/shop app.orders");
+  });
+
+  test("with owned, a foreign object is filtered, never absent", async () => {
+    const r = normalizeObservation(await describeResources(options(DECLARED, true)));
+    expect(r.unobserved.legacy).toMatchObject({ reason: "filtered" });
+    expect(r.unobserved.migrations?.detail).toMatch(/keeps this object/);
+  });
+});
+
+describeObservationConformance({
+  lexicon: "sql",
+  ownershipChannel: sqlPlugin.ownershipChannel,
+  scenarios: [
+    {
+      name: "Postgres: owned, foreign and absent",
+      declared: ["users", "legacy", "orders"],
+      expectPresent: ["users", "legacy"],
+      expectAbsent: ["orders"],
+      run: () => describeResources(options({ users: DECLARED.users, legacy: DECLARED.legacy, orders: DECLARED.orders })),
+    },
+    {
+      name: "Postgres: owned only",
+      declared: ["users", "legacy"],
+      owned: true,
+      expectPresent: ["users"],
+      expectUnobserved: ["legacy"],
+      expectMarker: { users: { stack: "shop", env: "prod" } },
+      run: () => describeResources(options({ users: DECLARED.users, legacy: DECLARED.legacy }, true)),
+    },
+    {
+      name: "Postgres: a server that refuses the password",
+      declared: ["users"],
+      expectUnobserved: ["users"],
+      run: () =>
+        describeResources({
+          ...options({ users: DECLARED.users }),
+          connect: async () => {
+            throw new PostgresQueryError("password authentication failed for user", "28P01");
+          },
+        }),
+    },
+    {
+      name: "Postgres: nothing bound",
+      declared: ["users"],
+      expectUnobserved: ["users"],
+      run: () => describeResources({ ...options({ users: DECLARED.users }), environment: "staging", env: {} }),
+    },
+  ],
+});
+
+describe("a managed provider's own objects (sql.provider)", () => {
+  const live: LivePgObject[] = [
+    { type: "Postgres::Schema", name: "auth", oid: "1", statement: "CREATE SCHEMA auth" },
+    { type: "Postgres::Table", schema: "auth", name: "users", oid: "2", statement: "CREATE TABLE auth.users (id uuid)" },
+    { type: "Postgres::Extension", name: "pg_net", oid: "3", statement: "CREATE EXTENSION pg_net WITH SCHEMA extensions" },
+    { type: "Postgres::Table", schema: "app", name: "orders", oid: "4", statement: "CREATE TABLE app.orders (id bigint)" },
+  ];
+
+  test("on Supabase, its schemas, what is in them and its extensions read as Supabase's", () => {
+    const marked = markProviderOwned(live, "supabase");
+    expect(marked.map((o) => [o.name, o.foreign])).toEqual([
+      ["auth", "Supabase"],
+      ["users", "Supabase"],
+      ["pg_net", "Supabase"],
+      ["orders", undefined],
+    ]);
+    expect(markProviderOwned(live, undefined).every((o) => o.foreign === undefined)).toBe(true);
+  });
+
+  test("the provider comes from the profile, else the namespace", () => {
+    expect(resolvePostgresTarget({ environment: "s", config: { sql: { provider: "supabase", profiles: { s: { url: "postgres://x/y" } } } }, env: {} })).toMatchObject({ provider: "supabase" });
+    expect(resolvePostgresTarget({ environment: "s", config: { sql: { provider: "rds", profiles: { s: { url: "postgres://x/y", provider: "neon" } } } }, env: {} })).toMatchObject({ provider: "neon" });
+  });
+
+  test("describeResources reports a declared object in a provider schema as foreign", async () => {
+    const r = normalizeObservation(
+      await describeResources({
+        environment: "prod",
+        config: { sql: { provider: "supabase", profiles: { prod: { url: "postgres://db/shop" } } } },
+        env: {},
+        entityNames: ["authUsers"],
+        entities: new Map([["authUsers", entity("Postgres::Table", { schema: "auth", name: "users" })]]),
+        connect: async () => fakeClient({ tables: [{ schema: "auth", name: "users", comment: "[chant managed-by=chant]" }] }),
+      }),
+    );
+    expect(r.resources.authUsers).toMatchObject({ ownership: "foreign", attributes: { keptBy: "Supabase" } });
+  });
+});

package/src/postgres/live/roundtrip.e2e.test.ts

@@ -0,0 +1,111 @@
+/**
+ * The live round trip (#3279): a schema on one Postgres database is imported
+ * as chant declarations, built, and applied to a second, empty database;
+ * both catalogs then print the same statements for every object, exporting
+ * the second gives back the same declarations, every declared object is
+ * present, and the deep read of the applied database reports no drift from
+ * the declarations.
+ *
+ * Needs Docker; skips cleanly without it. One throwaway `postgres` container
+ * at the pin (18.6, by tag and digest), removed afterwards even on failure.
+ */
+
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { build } from "@intentius/chant/build";
+import type { SerializerResult } from "@intentius/chant/serializer";
+import { dockerAvailable, startTestPostgres, type TestPostgres } from "../testing/server";
+import { ROUNDTRIP_SCHEMA } from "../testing/roundtrip-schema";
+import { exportResources } from "../import/live-export";
+import { PostgresGenerator } from "../import/generator";
+import { readLiveSchema } from "./catalog";
+import { describeResources } from "./describe-resources";
+import { observeResourcesDeep } from "../plan/deep";
+import { sqlPlugin } from "../../plugin";
+import { POSTGRES_DDL_FILE, sqlSerializer } from "../../serializer";
+
+const enabled = await dockerAvailable();
+const projectDir = join(import.meta.dirname, "..", "..", "..", `.pg-roundtrip-tmp-${process.pid}`);
+let server: TestPostgres | undefined;
+
+beforeAll(async () => {
+  if (!enabled) return;
+  server = await startTestPostgres();
+  const admin = await server.connect();
+  await admin.query("CREATE DATABASE source");
+  await admin.query("CREATE DATABASE target");
+  await admin.end();
+  const source = await server.connect("source");
+  await source.query(ROUNDTRIP_SCHEMA);
+  await source.end();
+}, 600_000);
+
+afterAll(async () => {
+  await server?.stop();
+  rmSync(projectDir, { recursive: true, force: true });
+});
+
+const profile = (db: string) => ({
+  config: { sql: { profiles: { rt: { url: server!.endpoint(db).url, password: { env: "PG_RT_PASSWORD" } } } } },
+  env: { PG_RT_PASSWORD: server!.endpoint(db).password },
+});
+
+/** The props a deep diff compares: the declaration's, less what the hooks prune. */
+const PRUNED = new Set(["ddl", "source", "lineage", "reads", "concurrently", "orReplace", "ifNotExists"]);
+const compared = (props: Record<string, unknown>) =>
+  JSON.parse(JSON.stringify(Object.fromEntries(Object.entries(props).filter(([k]) => !PRUNED.has(k))), (_k, v: unknown) =>
+    v !== null && typeof v === "object" && "sqlName" in (v as object) && "entityType" in (v as object) ? `ref:${(v as { sqlName: string }).sqlName}` : v,
+  )) as Record<string, unknown>;
+
+async function catalog(db: string): Promise<Record<string, string>> {
+  const client = await server!.connect(db);
+  try {
+    return Object.fromEntries((await readLiveSchema(client)).filter((o) => !o.foreign).map((o) => [`${o.type} ${o.schema ?? ""}.${o.name}`, o.statement]));
+  } finally {
+    await client.end();
+  }
+}
+
+describe.skipIf(!enabled)("import, build, apply, and the catalog is unchanged", () => {
+  test("the round trip", async () => {
+    // Import: the source database's schema as declarations. The ORM's table is left out, with a warning.
+    const exported = await exportResources({ environment: "rt", ...profile("source") });
+    expect(exported.warnings?.some((w) => /_prisma_migrations is kept by Prisma Migrate/.test(w))).toBe(true);
+    const [file] = new PostgresGenerator().generate(exported);
+    mkdirSync(join(projectDir, "src"), { recursive: true });
+    writeFileSync(join(projectDir, "chant.config.ts"), 'export default { lexicons: ["sql"] };\n');
+    writeFileSync(join(projectDir, "src", file!.path), file!.content);
+
+    // Build: the declarations fold to the statements.
+    const result = await build(join(projectDir, "src"), [sqlSerializer], undefined, { fold: true, intrinsics: sqlPlugin.intrinsics!(), lexicons: ["sql"] });
+    expect(result.errors).toEqual([]);
+    const out = result.outputs.get("sql") as SerializerResult;
+
+    // Apply: run the statements, in the order the build wrote them, on the empty database.
+    const target = await server!.connect("target");
+    await target.query(out.files![POSTGRES_DDL_FILE]!);
+    await target.end();
+
+    // The catalog is unchanged.
+    expect(await catalog("target")).toEqual(await catalog("source"));
+
+    // Exporting the applied database gives back the same declarations.
+    const again = await exportResources({ environment: "rt", ...profile("target") });
+    expect(new PostgresGenerator().generate(again)[0]!.content).toBe(file!.content);
+
+    // Every declared object is present: nothing to create.
+    const entities = new Map(
+      [...result.entities].map(([k, v]) => [k, { entityType: v.entityType, props: (v as unknown as { props: Record<string, unknown> }).props }]),
+    );
+    const observed = await describeResources({ environment: "rt", entityNames: [...entities.keys()], entities, ...profile("target") });
+    expect(Object.keys((observed as { resources: object }).resources).sort()).toEqual([...entities.keys()].sort());
+
+    // The deep read reports no property drift from the declarations.
+    const deep = await observeResourcesDeep({ environment: "rt", entityNames: [...entities.keys()], entities, ...profile("target") });
+    expect(deep.unobserved ?? {}).toEqual({});
+    for (const [name, e] of entities) {
+      expect(compared(deep.resources[name]!.properties), name).toEqual(JSON.parse(JSON.stringify(compared(e.props))));
+    }
+  }, 600_000);
+});

package/src/postgres/migrate/backfill.ts

@@ -0,0 +1,233 @@
+/**
+ * The backfill and the verification of the expand-and-contract migration
+ * (#3281).
+ *
+ * ## Backfill
+ *
+ * The rows that were in the table before the dual write started are filled
+ * in batches: one range of the primary key per batch (`./batches.ts`:
+ * `[b * size, (b + 1) * size)` of an integer key, else between boundaries
+ * recorded in the receipt store), so a batch is the same rows on every run
+ * however many rows were written since. Each batch is one transaction:
+ *
+ *     UPDATE t SET new = <expression> WHERE key >= lo AND key < hi AND new IS DISTINCT FROM <expression>;
+ *     INSERT INTO __chant_receipts ... -- the batch's receipt, last
+ *     COMMIT;
+ *
+ * under `lock_timeout` (a row an application transaction holds makes the
+ * batch give up and try again rather than queue behind it) and the
+ * statement timeout for a catalog change, which bounds how long one batch
+ * may hold its row locks.
+ *
+ * Per batch, the cycle core's `effect()` step runs
+ * (concepts/effect-receipts.mdx): read the receipt, skip the batch when it
+ * matches, otherwise update and write the receipt last, on success only.
+ * The batches are only known when the run reads the table, so the cycle runs
+ * here over the shared core's `ReceiptStore` seam (`./receipts.ts`), as the
+ * ClickHouse rebuild's does per partition. Unlike ClickHouse, the receipt
+ * commits in the batch's own transaction: a run killed anywhere leaves a
+ * batch with its receipt or neither, and a resumed backfill starts at the
+ * first batch with no receipt and updates no batch twice.
+ *
+ * Before each batch the backfill waits while a replica is further behind
+ * than the bound (`./replication.ts`).
+ *
+ * ## Verification
+ *
+ * One statement over the whole table, one snapshot: the rows, the rows whose
+ * new column differs from the expression over the old one, the new column's
+ * NULLs, and a checksum (the sum of a 64-bit hash of each value) of the new
+ * column and of the expression. Any difference, or a NULL where the column
+ * is declared NOT NULL, fails the run, whose onFailure then drops what the
+ * expand added. The dual-write trigger keeps writes going during the
+ * verification consistent; the statement reads the table under ACCESS SHARE
+ * only.
+ */
+
+import { EffectReceipt, receiptExpectation } from "@intentius/chant/effect-receipt";
+import { computePlanDigest } from "@intentius/chant/op";
+import { col } from "./names";
+import { MigrationRefusal, migrationPlanSubject, type MigrationObservation } from "./observe";
+import { postgresReceiptStore, receiptAddress, type PostgresReceiptStore } from "./receipts";
+import { waitForReplicas } from "./replication";
+import { batchPrefix, identityOf, inTransaction, observe, type MigrationRun } from "./steps";
+import { carriedNotReady, requiresNotNull } from "./carry";
+import { integerBatches, isIntegerKey, recordedBatches, type Batch } from "./batches";
+
+export interface BackfillResult {
+  state: MigrationObservation["state"];
+  /** Batches of the table's key ranges that hold rows. */
+  batches: number;
+  /** Batches filled in this run. */
+  filled: number;
+  /** Batches skipped: their receipt matched. */
+  skipped: number;
+  /** Rows this run updated. */
+  rows: number;
+  /** How long the backfill paused for replicas, in ms. */
+  pausedMs: number;
+}
+
+/** The effect a batch is, and its receipt's address suffix. */
+export const batchEffect = (key: string, batch: string): string => `migrate/${key}/${batch}`;
+
+/** The receipt's expected value for one batch: bound to the table, the new column's attribute number, the batch's rows and the expression. */
+export function batchExpectation(o: MigrationObservation, batch: Pick<Batch, "id" | "inputs">): string {
+  const effect = batchEffect(o.names.key, batch.id);
+  return receiptExpectation(
+    EffectReceipt(effect, {
+      effect,
+      flavor: "hash",
+      inputs: { migration: o.names.key, table: o.oid, column: o.newColumn!.attnum, ...batch.inputs, expression: o.expression },
+    }),
+  );
+}
+
+/** The address suffix of the recorded batch boundaries of a key that is not one integer column. */
+export const BATCHES_RECORD = "batches";
+
+export interface BackfillDeps {
+  /** The receipt store. Default: `<schema>.__chant_receipts` over the run's connection. */
+  receipts?: PostgresReceiptStore;
+  /** Called after a batch's receipt is written and before its transaction commits; a test kills the run here. */
+  beforeCommit?: (batch: string) => void | Promise<void>;
+  /** Called after each batch commits; a test interrupts the backfill here. */
+  afterBatch?: (batch: string) => void | Promise<void>;
+  /** Wait for replicas with this (tests). */
+  waitForReplicas?: typeof waitForReplicas;
+}
+
+export async function backfill(run: MigrationRun, deps: BackfillDeps = {}): Promise<BackfillResult> {
+  const o = await observe(run);
+  const result: BackfillResult = { state: o.state, batches: 0, filled: 0, skipped: 0, rows: 0, pausedMs: 0 };
+  if (o.state !== "migrate") return result;
+  const n = o.names;
+  if (!o.newColumn) throw new MigrationRefusal(`${n.key}: the new column is not there yet; the Expand phase adds it`);
+  if (!o.trigger) throw new MigrationRefusal(`${n.key}: the dual write is not on; the Dual write phase starts it, and the backfill fills only rows written before it`);
+  if (!Number.isInteger(run.batchSize) || run.batchSize < 1) throw new MigrationRefusal(`${n.key}: batchSize must be a positive integer, got ${run.batchSize}`);
+
+  const identity = identityOf(run.marker);
+  const receipts = deps.receipts ?? postgresReceiptStore(run.client, n.schema, identity, run.runId ? { runId: run.runId } : {});
+  await receipts.ensure();
+  const recorded = await receipts.readAll(batchPrefix(identity, n.key));
+
+  const batches = isIntegerKey(o.keyColumns)
+    ? await integerBatches(run.client, n.qualifiedTable, o.keyColumns[0]!, run.batchSize)
+    : (
+        await recordedBatches(run.client, {
+          table: n.qualifiedTable,
+          oid: o.oid,
+          attnum: o.newColumn.attnum,
+          keys: o.keyColumns,
+          size: run.batchSize,
+          effect: batchEffect(n.key, BATCHES_RECORD),
+          receipts,
+          log: run.log,
+          ...(run.signal ? { signal: run.signal } : {}),
+        })
+      ).batches;
+  result.batches = batches.length;
+  const wait = deps.waitForReplicas ?? waitForReplicas;
+  const target = `${col(n.newColumn)}`;
+
+  for (const b of batches) {
+    run.signal?.throwIfAborted();
+    const effect = batchEffect(n.key, b.id);
+    const expectation = batchExpectation(o, b);
+    if (recorded.get(receiptAddress(identity, effect)) === expectation) {
+      result.skipped++;
+      continue;
+    }
+    if (run.replicationLag) result.pausedMs += await wait(run.client, run.replicationLag, { log: run.log, ...(run.signal ? { signal: run.signal } : {}) });
+    const updated = await inTransaction(run, async (exec) => {
+      const [row] = await exec(
+        `WITH u AS (UPDATE ${n.qualifiedTable} SET ${target} = (${o.expression}) WHERE ${b.where} AND ${target} IS DISTINCT FROM (${o.expression}) RETURNING 1) ` +
+          "SELECT count(*)::int AS n FROM u",
+        b.params,
+      );
+      // The receipt, last, in the batch's own transaction.
+      await receipts.write({ name: effect, effect, flavor: "hash", inputs: {} }, expectation);
+      await deps.beforeCommit?.(b.id);
+      return Number(row?.n ?? 0);
+    });
+    result.rows += updated;
+    result.filled++;
+    await deps.afterBatch?.(b.id);
+  }
+  run.log(`-- backfill of ${n.key}: ${result.batches} batch(es), ${result.filled} filled (${result.rows} row(s)), ${result.skipped} already filled`);
+  return result;
+}
+
+// ── verify ─────────────────────────────────────────────────────────────
+
+export interface VerifyResult {
+  state: MigrationObservation["state"];
+  /** The digest the switch gate binds: the plan and the verified new column. Absent once the switch has run. */
+  planDigest?: string;
+  rows: number;
+  /** Rows whose new column differs from the expression over the old one. */
+  mismatched: number;
+  /** Rows whose new column is NULL. */
+  nulls: number;
+  /** The sum of a 64-bit hash of every new value, and of every value the expression gives. */
+  checksum: string;
+  expected: string;
+  /** One line for the run record and the gate. */
+  summary: string;
+}
+
+/** A verification that found the columns different. */
+export class MigrationVerificationError extends Error {
+  constructor(
+    readonly key: string,
+    readonly result: Omit<VerifyResult, "summary" | "planDigest">,
+    reason: string,
+  ) {
+    super(`${key}: ${reason}. Nothing was switched; onFailure drops what the expand added, and the next run starts again.`);
+    this.name = "MigrationVerificationError";
+  }
+}
+
+export async function verifyMigration(run: MigrationRun): Promise<VerifyResult> {
+  const o = await observe(run);
+  const empty = { rows: 0, mismatched: 0, nulls: 0, checksum: "0", expected: "0" };
+  if (o.state !== "migrate") return { state: o.state, ...empty, summary: `nothing to verify: ${o.state}` };
+  const n = o.names;
+  if (!o.newColumn || !o.trigger) throw new MigrationRefusal(`${n.key}: the new column and its dual write are not there; the Expand and Dual write phases make them`);
+  const nw = col(n.newColumn);
+  const [row] = await inTransaction(
+    run,
+    (exec) =>
+      exec(
+        `SELECT count(*)::text AS rows, count(*) FILTER (WHERE ${nw} IS DISTINCT FROM (${o.expression}))::text AS mismatched,
+                count(*) FILTER (WHERE ${nw} IS NULL)::text AS nulls,
+                COALESCE(sum(pg_catalog.hashtextextended(${nw}::text, 0)), 0)::text AS checksum,
+                COALESCE(sum(pg_catalog.hashtextextended((${o.expression})::text, 0)), 0)::text AS expected
+         FROM ${n.qualifiedTable}`,
+      ),
+    { scan: true },
+  );
+  const r = { rows: Number(row?.rows ?? 0), mismatched: Number(row?.mismatched ?? 0), nulls: Number(row?.nulls ?? 0), checksum: String(row?.checksum ?? "0"), expected: String(row?.expected ?? "0") };
+  if (r.mismatched > 0 || r.checksum !== r.expected) {
+    throw new MigrationVerificationError(n.key, { state: o.state, ...r }, `${r.mismatched} of ${r.rows} row(s) have ${n.newColumn} different from ${o.expression} (checksums ${r.checksum} and ${r.expected})`);
+  }
+  if (requiresNotNull(o.column, o.carried) && r.nulls > 0) {
+    throw new MigrationVerificationError(n.key, { state: o.state, ...r }, `${r.nulls} of ${r.rows} row(s) have ${n.newColumn} NULL, and ${n.column} is declared NOT NULL or is in the primary key`);
+  }
+  const notReady = carriedNotReady(o.carried, o.carriedStates);
+  if (notReady.length > 0) {
+    throw new MigrationRefusal(`${n.key}: ${notReady.map((c) => `${c.kind} ${c.working}`).join(", ")} on the new column is not there or not yet valid; the Carry over phase makes them`);
+  }
+  const carried =
+    o.carried.length > 0 || o.views.length > 0
+      ? `; ${o.carried.map((c) => `${c.kind} ${c.name}${c.target !== c.name ? ` (as ${c.target})` : ""}`).join(", ") || "no index or constraint"} carried over${o.views.length > 0 ? `, view(s) ${o.views.map((v) => v.name).join(", ")} made again at the switch` : ""}`
+      : "";
+  const summary = `${n.key}: ${r.rows} row(s), ${n.newColumn} equal to ${o.expression} in every one (checksum ${r.checksum})${carried}`;
+  run.log(`-- ${summary}`);
+  return {
+    state: o.state,
+    planDigest: computePlanDigest("postgres-migration", { plan: migrationPlanSubject(o), table: o.oid, column: o.newColumn.attnum, verified: { mismatched: 0 } }),
+    ...r,
+    summary,
+  };
+}