@intentius/chant-lexicon-sql 0.114.0 → 0.115.0

package/src/postgres/plan/deep.test.ts

@@ -0,0 +1,76 @@
+/**
+ * The Postgres deep reader's columns (INTENTIUS/sql-yodeler#47): matched by
+ * name, not by position, because Postgres keeps a table's columns in the
+ * order they were added and a column renamed by expand and contract is last
+ * on the server.
+ */
+
+import { describe, expect, test } from "vitest";
+import * as pg from "../entities";
+import { POSTGRES_ENTITY_TYPES } from "../entity-types";
+import type { LivePgObject } from "../live/catalog";
+import { inDeclaredVocabulary, liveProps } from "./deep";
+import { canonicalPgObject } from "./normalize";
+
+type Props = Record<string, unknown>;
+
+const declared = (pg.table`
+  CREATE TABLE shop.orders (
+      id bigint GENERATED ALWAYS AS IDENTITY,
+      email text NOT NULL, -- previously: customer_email
+      status text DEFAULT 'placed'::text NOT NULL,
+      amount numeric(12,2) NOT NULL,
+      refunded_at timestamp with time zone,
+      CONSTRAINT orders_pkey PRIMARY KEY (id)
+  )` as unknown as { props: Props }).props;
+
+/** What the deep reader makes of the server's definition of the table. */
+function observe(statement: string): Props {
+  const o = { type: POSTGRES_ENTITY_TYPES.table, schema: "shop", name: "orders", oid: "1", statement } as unknown as LivePgObject;
+  const live = liveProps(o);
+  return inDeclaredVocabulary(declared, live, canonicalPgObject(POSTGRES_ENTITY_TYPES.table, declared), canonicalPgObject(POSTGRES_ENTITY_TYPES.table, live));
+}
+
+const names = (p: Props) => (p.columns as Array<{ name: string }>).map((c) => c.name);
+
+describe("Postgres deep observation of a table's columns", () => {
+  test("a column the server holds last and the declaration earlier is not drift", () => {
+    const props = observe(`CREATE TABLE shop.orders (
+    id bigint GENERATED ALWAYS AS IDENTITY NOT NULL,
+    status text DEFAULT 'placed'::text NOT NULL,
+    amount numeric(12,2) NOT NULL,
+    refunded_at timestamp with time zone,
+    email text NOT NULL,
+    CONSTRAINT orders_pkey PRIMARY KEY (id)
+)`);
+    expect(props.columns).toEqual(declared.columns);
+  });
+
+  test("a column that differs is reported at its declared place, and an undeclared one comes after the declared ones", () => {
+    const props = observe(`CREATE TABLE shop.orders (
+    id bigint GENERATED ALWAYS AS IDENTITY NOT NULL,
+    note text,
+    status text DEFAULT 'placed'::text NOT NULL,
+    amount numeric(12,2) NOT NULL,
+    refunded_at timestamp with time zone,
+    email character varying(200) NOT NULL,
+    CONSTRAINT orders_pkey PRIMARY KEY (id)
+)`);
+    expect(names(props)).toEqual(["id", "email", "status", "amount", "refunded_at", "note"]);
+    const columns = props.columns as Props[];
+    const declaredColumns = declared.columns as Props[];
+    expect(columns[1]).not.toEqual(declaredColumns[1]);
+    expect(columns.slice(2, 5)).toEqual(declaredColumns.slice(2, 5));
+  });
+
+  test("a declared column the server does not have leaves the others matched by name", () => {
+    const props = observe(`CREATE TABLE shop.orders (
+    id bigint GENERATED ALWAYS AS IDENTITY NOT NULL,
+    status text DEFAULT 'placed'::text NOT NULL,
+    amount numeric(12,2) NOT NULL,
+    email text NOT NULL,
+    CONSTRAINT orders_pkey PRIMARY KEY (id)
+)`);
+    expect(names(props)).toEqual(["id", "email", "status", "amount"]);
+  });
+});

package/src/postgres/plan/deep.ts

@@ -108,11 +108,22 @@ export function inDeclaredVocabulary(declared: Props, live: Props, d: CanonicalP
   if (d.kind === "table") {
     const declaredColumns = (declared.columns as unknown[] | undefined) ?? [];
     const liveColumns = (live.columns as unknown[] | undefined) ?? [];
-    out.columns = liveColumns.map((c, i) => {
+    // Columns are matched by name and listed in the declaration's order, the
+    // live columns the declaration does not name after them. Postgres keeps a
+    // table's columns in the order they were added, and no ALTER moves one: a
+    // column renamed by expand and contract is last on the server and where
+    // the old one was in the declaration (INTENTIUS/sql-yodeler#47). The plan
+    // compares columns by name too, so an order the server cannot be brought
+    // to is not drift.
+    const read = liveColumns.map((c, i) => {
       const lc = l.columns[i];
       const dc = lc ? d.columns.find((x) => x.name === lc.name) : undefined;
-      return lc && dc && columnShape(lc) === columnShape(dc) ? declaredColumns[dc.position] : c;
+      return { at: dc?.position, value: lc && dc && columnShape(lc) === columnShape(dc) ? declaredColumns[dc.position] : c };
     });
+    out.columns = [
+      ...read.filter((c) => c.at !== undefined).sort((a, b) => a.at! - b.at!),
+      ...read.filter((c) => c.at === undefined),
+    ].map((c) => c.value);
   }
   if (d.kind === "table" || d.kind === "domain") {
     // A constraint the declaration says the same as the server is the declaration's (its name included or left out).

package/src/postgres/plan/plan.e2e.test.ts

@@ -91,6 +91,32 @@ describe.skipIf(!enabled)("planning the getting-started example against a server
     expect((await planPgAgainstServer("e2e", file, profile("star"))).changes.map((c) => `${c.field}: ${c.before} -> ${c.after}`)).toEqual([]);
   }, 600_000);
 
+  test("a column renamed by expand and contract, last on the server, plans and reads back with no drift (sql-yodeler#47)", async () => {
+    const admin = await server!.connect();
+    await admin.query("CREATE DATABASE renamed");
+    await admin.end();
+    const orders = pg.table`CREATE TABLE public.orders (
+      id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+      email text NOT NULL, -- previously: customer_email
+      status text DEFAULT 'placed'::text NOT NULL,
+      refunded_at timestamp with time zone
+    )`;
+    const out = sqlSerializer.serialize(new Map<string, unknown>([["orders", orders]]) as never) as SerializerResult;
+    const file = join(dir, "renamed.json");
+    writeFileSync(file, out.primary);
+    const client = await server!.connect("renamed");
+    await client.query(
+      "CREATE TABLE public.orders (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, customer_email text NOT NULL, status text DEFAULT 'placed'::text NOT NULL, refunded_at timestamp with time zone);" +
+        "ALTER TABLE public.orders ADD COLUMN email text; UPDATE public.orders SET email = customer_email; ALTER TABLE public.orders ALTER COLUMN email SET NOT NULL; ALTER TABLE public.orders DROP COLUMN customer_email",
+    );
+    expect((await planPgAgainstServer("e2e", file, profile("renamed"))).changes).toEqual([]);
+
+    const props = (orders as unknown as { props: Record<string, unknown> }).props;
+    const deep = await observeResourcesDeep({ environment: "e2e", entityNames: ["orders"], entities: new Map([["orders", { entityType: "Postgres::Table", props }]]), ...profile("renamed") });
+    expect(deep.resources.orders!.properties.columns).toEqual(props.columns);
+    await client.end();
+  }, 600_000);
+
   test("a serial table plans with no changes after apply; serial to bigserial is integer to bigint, unqualified", async () => {
     const admin = await server!.connect();
     await admin.query("CREATE DATABASE serials");

package/src/receipts.test.ts

@@ -0,0 +1,91 @@
+import { describe, expect, test } from "vitest";
+import { environmentOf, receiptDialect, sqlReceiptStore } from "./receipts";
+import type { PostgresClient } from "./postgres/live/client";
+
+const ref = (effect: string) => ({ name: effect, effect, flavor: "existence" as const, inputs: {} });
+
+/** A Postgres client that records each statement and answers the store's catalog reads. */
+function fakePostgres(state: { schema?: boolean; table?: string | null } = {}) {
+  const statements: string[] = [];
+  let schema = state.schema ?? false;
+  let table: string | null | undefined = state.table;
+  const rows = new Map<string, string>();
+  const client: PostgresClient = {
+    async query<T>(sql: string, params: readonly unknown[] = []): Promise<T[]> {
+      statements.push(sql);
+      if (sql.includes("pg_namespace WHERE nspname")) return [{ n: schema ? 1 : 0 }] as T[];
+      if (sql.startsWith("CREATE SCHEMA")) schema = true;
+      if (sql.includes("to_regclass($1) IS NOT NULL AS present")) return [{ present: table !== undefined, comment: table ?? null }] as T[];
+      if (sql.startsWith("CREATE TABLE")) table = null;
+      if (sql.startsWith("COMMENT ON TABLE")) table = sql.slice(sql.indexOf("IS '") + 4, -1);
+      if (sql.startsWith("INSERT INTO")) rows.set(String(params[0]), String(params[2]));
+      if (sql.startsWith("SELECT expectation")) {
+        const v = rows.get(String(params[0]));
+        return (v === undefined ? [] : [{ expectation: v }]) as T[];
+      }
+      return [] as T[];
+    },
+    end: async () => undefined,
+  };
+  return { client, statements, rows };
+}
+
+describe("receiptDialect (#3657)", () => {
+  test("the environment's profile decides, by its URL's scheme", () => {
+    const config = { sql: { profiles: { pg: { url: "postgres://db:5432/app" }, ch: { url: "http://ch:8123" } } } };
+    expect(receiptDialect(config, "pg", {})).toBe("postgres");
+    expect(receiptDialect(config, "ch", {})).toBe("clickhouse");
+  });
+
+  test("with no profile, sql.dialect, then whichever server variable is set", () => {
+    expect(receiptDialect({ sql: { dialect: "postgres" } }, "dev", {})).toBe("postgres");
+    expect(receiptDialect({ sql: { dialect: ["clickhouse"] } }, undefined, { POSTGRES_URL: "postgres://x/y" })).toBe("clickhouse");
+    expect(receiptDialect(undefined, undefined, { POSTGRES_URL: "postgres://x/y" })).toBe("postgres");
+    expect(receiptDialect(undefined, undefined, {})).toBe("clickhouse");
+  });
+});
+
+describe("environmentOf (#3657)", () => {
+  test("the option, then CHANT_ENV, then a literal ownership.env", () => {
+    expect(environmentOf({ environment: "a" }, { ownership: { stack: "s", env: "c" } }, { CHANT_ENV: "b" })).toBe("a");
+    expect(environmentOf({}, { ownership: { stack: "s", env: "c" } }, { CHANT_ENV: "b" })).toBe("b");
+    expect(environmentOf({}, { ownership: { stack: "s", env: "c" } }, {})).toBe("c");
+    expect(environmentOf({}, undefined, {})).toBeUndefined();
+  });
+});
+
+describe("sqlReceiptStore over Postgres (#3657)", () => {
+  test("keeps receipts in chant_receipts.receipts, a schema and table it marks as chant's", async () => {
+    const pg = fakePostgres();
+    const store = sqlReceiptStore({ environment: "prod", stack: "shop", postgres: pg.client, runId: "r1" });
+    expect(await store.read(ref("fill/b1"))).toBeUndefined();
+    await store.write(ref("fill/b1"), "present");
+    expect(await store.read(ref("fill/b1"))).toBe("present");
+
+    expect(pg.statements).toContain("CREATE SCHEMA IF NOT EXISTS chant_receipts");
+    expect(pg.statements.find((s) => s.startsWith("COMMENT ON SCHEMA chant_receipts"))).toContain("receipts=effects");
+    expect(pg.statements.find((s) => s.startsWith("CREATE TABLE"))).toContain("TABLE IF NOT EXISTS chant_receipts.receipts (");
+    expect([...pg.rows.keys()]).toEqual(["shop/prod/fill/b1"]);
+    expect(await store.location()).toEqual({ dialect: "postgres", table: "chant_receipts.receipts", source: "the caller's connection", address: "shop/prod/<effect>" });
+  });
+
+  test("a receiptsSchema keeps them in <schema>.__chant_receipts, and creates no schema", async () => {
+    const pg = fakePostgres();
+    const store = sqlReceiptStore({
+      environment: "prod",
+      postgres: pg.client,
+      config: { ownership: { stack: "shop" }, sql: { profiles: { prod: { url: "postgres://db/app", receiptsSchema: "app" } } } },
+    });
+    await store.write(ref("fill/b1"), "present");
+    expect(pg.statements.some((s) => s.includes("CREATE SCHEMA"))).toBe(false);
+    expect(pg.statements.find((s) => s.startsWith("CREATE TABLE"))).toContain("TABLE IF NOT EXISTS app.__chant_receipts (");
+    expect((await store.location()).table).toBe("app.__chant_receipts");
+  });
+
+  test("a table of the user's under the receipts name is refused, not written into", async () => {
+    const pg = fakePostgres({ schema: true, table: "the user's own receipts" });
+    const store = sqlReceiptStore({ environment: "prod", stack: "shop", postgres: pg.client });
+    await expect(store.write(ref("fill/b1"), "present")).rejects.toThrow(/chant_receipts\.receipts exists and is not chant's receipts table/);
+    expect(pg.rows.size).toBe(0);
+  });
+});

package/src/receipts.ts

@@ -0,0 +1,259 @@
+/**
+ * The sql lexicon's `effect()` receipt store (#3657): where the receipts of a
+ * project that uses only the sql lexicon are kept, on the database server the
+ * environment names.
+ *
+ *     ClickHouse   chant_receipts.receipts
+ *     Postgres     chant_receipts.receipts   (or <receiptsSchema>.__chant_receipts)
+ *
+ * The op activities module binds it as `receiptRead`, `receiptWrite` and
+ * `receiptStaleness` (`./op/activities/index.ts`), listed as fallbacks: a
+ * project that also configures aws or k8s keeps its receipts in that
+ * lexicon's receipt row, in whichever order `lexicons` lists them.
+ *
+ * Which server: the environment is the one `chant run --env` sets
+ * (`CHANT_ENV`), else a literal `ownership.env`. Its `sql.profiles.<env>`
+ * binds the server, a `postgres://` URL making it Postgres; with no profile,
+ * `CLICKHOUSE_URL` or `POSTGRES_URL`, picked by `sql.dialect`. A receipt's
+ * address is `<stack>/<env>/<effect>`, from `ownership.stack` and that
+ * environment, the same identity the rebuild's and the column migration's
+ * receipts carry (`./core/receipts.ts`).
+ *
+ * Both tables carry chant's ownership trailer with the `receipts` key in their
+ * comment (the Postgres schema too), so plan, import and prune leave them out.
+ *
+ * A caller that runs its own Op over a server it has already bound (a
+ * migration runner, say) passes that server instead: a ClickHouse endpoint,
+ * or a Postgres connection, whose receipt writes then commit or roll back
+ * with whatever transaction the caller has open on it.
+ */
+
+import type { ChantConfig } from "@intentius/chant/config";
+import { receiptActivities, type ReceiptActivities, type ReceiptActivityOptions } from "@intentius/chant/op/receipt-store";
+import { OWNERSHIP_MANAGED_BY_VALUE } from "@intentius/chant/ownership";
+import type { SqlReceiptStore } from "./core/receipts";
+import { COMMENT_OWNERSHIP_KEYS, RECEIPTS_TRAILER_KEY } from "./core/ownership";
+import type { ClickHouseEndpoint } from "./clickhouse/http";
+import type { Topology } from "./clickhouse/topology";
+import type { PostgresClient } from "./postgres/live/client";
+
+/** The database (ClickHouse) or schema (Postgres) the receipts are kept in. */
+export const SQL_RECEIPTS_SCHEMA = "chant_receipts";
+/** The table in it. */
+export const SQL_RECEIPTS_TABLE = "receipts";
+/** The table's name in a schema of the user's own (`receiptsSchema`), as the column migration names it. */
+export const SQL_RECEIPTS_TABLE_BESIDE = "__chant_receipts";
+
+export interface SqlReceiptStoreOptions {
+  /** The chant environment. Default: `CHANT_ENV`, then a literal `ownership.env`. */
+  environment?: string;
+  /** The address's stack segment. Default: `ownership.stack`. */
+  stack?: string;
+  /** The project directory chant.config.ts is read from. Default: the working directory. */
+  cwd?: string;
+  /** The project's config, instead of reading chant.config.ts. */
+  config?: Pick<ChantConfig, "sql" | "ownership">;
+  /** The process environment the bindings and `CHANT_ENV` are read from. Default: `process.env`. */
+  env?: Record<string, string | undefined>;
+  /** A ClickHouse server already bound: the receipts go there, and no profile is read. */
+  clickhouse?: { endpoint: ClickHouseEndpoint; topology?: Topology };
+  /**
+   * A Postgres connection already open: the receipts go over it, and it is
+   * never closed. A write runs inside whatever transaction the caller has
+   * open, so it commits or rolls back with it. Call `ensure()` before such a
+   * transaction begins, so the receipts table does not roll back with it.
+   */
+  postgres?: PostgresClient;
+  /** Postgres: the schema to keep the receipts in. Default: the profile's `receiptsSchema`, then `chant_receipts`. */
+  schema?: string;
+  /** Recorded on each receipt written. Default: the current Op run's id. */
+  runId?: string;
+}
+
+/** The store, and the step a caller with its own transaction takes before one opens. */
+export interface SqlServerReceiptStore extends SqlReceiptStore {
+  /** Create the receipts table (and database or schema) if it is not there. */
+  ensure(): Promise<void>;
+  /** Where the receipts are, as resolved: `chant_receipts.receipts` and the server it is on. */
+  location(): Promise<{ dialect: "clickhouse" | "postgres"; table: string; source: string; address: string }>;
+}
+
+type Config = Pick<ChantConfig, "sql" | "ownership">;
+
+interface Resolved {
+  identity: { stack?: string; env?: string };
+  open(): Promise<{ store: SqlReceiptStore & { ensure?: () => Promise<void> }; close(): Promise<void> }>;
+  dialect: "clickhouse" | "postgres";
+  table: string;
+  source: string;
+}
+
+async function loadConfig(cwd: string): Promise<Config | undefined> {
+  try {
+    const { loadChantConfig } = await import("@intentius/chant/config");
+    return (await loadChantConfig(cwd)).config;
+  } catch {
+    return undefined;
+  }
+}
+
+const isPostgresUrl = (url: string): boolean => /^postgres(ql)?:\/\//i.test(url);
+
+/** The dialect an environment's receipts are kept in: its profile's URL, else `sql.dialect`, else whichever server variable is set. */
+export function receiptDialect(config: Config | undefined, environment: string | undefined, env: Record<string, string | undefined>): "clickhouse" | "postgres" {
+  const profile = environment !== undefined ? config?.sql?.profiles?.[environment] : undefined;
+  if (profile) return isPostgresUrl(profile.url) ? "postgres" : "clickhouse";
+  const dialects = config?.sql?.dialect === undefined ? [] : Array.isArray(config.sql.dialect) ? config.sql.dialect : [config.sql.dialect];
+  if (dialects.length === 1) return dialects[0] === "postgres" ? "postgres" : "clickhouse";
+  if (!env.CLICKHOUSE_URL && env.POSTGRES_URL) return "postgres";
+  return "clickhouse";
+}
+
+/** The run's environment: the option, `CHANT_ENV`, or a literal `ownership.env`. */
+export function environmentOf(options: { environment?: string }, config: Pick<ChantConfig, "ownership"> | undefined, env: Record<string, string | undefined>): string | undefined {
+  if (options.environment) return options.environment;
+  if (env.CHANT_ENV) return env.CHANT_ENV;
+  const literal = config?.ownership?.env;
+  return typeof literal === "string" ? literal : undefined;
+}
+
+async function currentRunId(): Promise<string | undefined> {
+  try {
+    const { currentOpRun } = await import("@intentius/chant/op");
+    return currentOpRun()?.runId;
+  } catch {
+    return undefined;
+  }
+}
+
+const receiptsComment = `chant effect receipts [chant ${COMMENT_OWNERSHIP_KEYS.managedBy}=${OWNERSHIP_MANAGED_BY_VALUE} ${RECEIPTS_TRAILER_KEY}=effects]`;
+
+async function resolve(options: SqlReceiptStoreOptions): Promise<Resolved> {
+  const env = options.env ?? process.env;
+  const needsConfig = options.config === undefined && (!options.stack || (!options.clickhouse && !options.postgres) || !(options.environment || env.CHANT_ENV));
+  const config = options.config ?? (needsConfig ? await loadConfig(options.cwd ?? process.cwd()) : undefined);
+  const environment = environmentOf(options, config, env);
+  const stack = options.stack ?? (config?.ownership && config.ownership.enabled !== false ? config.ownership.stack : undefined);
+  const identity = { ...(stack ? { stack } : {}), ...(environment ? { env: environment } : {}) };
+  const runId = options.runId ?? (await currentRunId());
+  const dialect = options.postgres ? "postgres" : options.clickhouse ? "clickhouse" : receiptDialect(config, environment, env);
+
+  if (dialect === "clickhouse") {
+    const { clickhouseReceiptStore } = await import("./clickhouse/rebuild/receipts");
+    let endpoint: ClickHouseEndpoint;
+    let topology: Topology | undefined;
+    let source: string;
+    if (options.clickhouse) {
+      endpoint = options.clickhouse.endpoint;
+      topology = options.clickhouse.topology;
+      source = endpoint.url;
+    } else {
+      const { bindClickHouse } = await import("./clickhouse/live/bind");
+      const target = await bindClickHouse({ ...(environment !== undefined ? { environment } : {}), config: config ?? {}, env });
+      endpoint = target.endpoint;
+      topology = target.topology;
+      source = target.source;
+    }
+    // Created for the environment's topology: ON CLUSTER on a cluster, a Replicated database in the replicated one.
+    const store = clickhouseReceiptStore(endpoint, identity, { ...(runId ? { runId } : {}), ...(topology ? { topology } : {}) });
+    return {
+      identity,
+      dialect,
+      table: `${SQL_RECEIPTS_SCHEMA}.${SQL_RECEIPTS_TABLE}`,
+      source,
+      open: async () => ({ store, close: async () => undefined }),
+    };
+  }
+
+  const { postgresReceiptStore } = await import("./postgres/migrate/receipts");
+  const profile = environment !== undefined ? config?.sql?.profiles?.[environment] : undefined;
+  const schema = options.schema ?? profile?.receiptsSchema ?? SQL_RECEIPTS_SCHEMA;
+  const own = schema === SQL_RECEIPTS_SCHEMA;
+  const table = own ? SQL_RECEIPTS_TABLE : SQL_RECEIPTS_TABLE_BESIDE;
+  const storeOver = (client: PostgresClient) => {
+    const store = postgresReceiptStore(client, schema, identity, { ...(runId ? { runId } : {}), table });
+    let ensured = false;
+    return {
+      ...store,
+      async ensure() {
+        if (ensured) return;
+        // chant's own schema is made here, marked so the catalog leaves it out; a schema of the user's must exist.
+        if (own) {
+          const [present] = await client.query<{ n: number }>("SELECT count(*)::int AS n FROM pg_catalog.pg_namespace WHERE nspname = $1", [SQL_RECEIPTS_SCHEMA]);
+          if (!present?.n) {
+            await client.query(`CREATE SCHEMA IF NOT EXISTS ${SQL_RECEIPTS_SCHEMA}`);
+            await client.query(`COMMENT ON SCHEMA ${SQL_RECEIPTS_SCHEMA} IS '${receiptsComment.replace(/'/g, "''")}'`);
+          }
+        }
+        await store.ensure();
+        ensured = true;
+      },
+    };
+  };
+  if (options.postgres) {
+    const client = options.postgres;
+    const store = storeOver(client);
+    return { identity, dialect, table: `${schema}.${table}`, source: "the caller's connection", open: async () => ({ store, close: async () => undefined }) };
+  }
+  const { resolveBoundTarget } = await import("./postgres/live/bind");
+  const { connectPostgres } = await import("./postgres/live/client");
+  const target = await resolveBoundTarget({ ...(environment !== undefined ? { environment } : {}), config: config ?? {}, env });
+  return {
+    identity,
+    dialect,
+    table: `${schema}.${table}`,
+    source: target.source,
+    // One connection per read or write: a run's process holds no connection open between steps, and exits when the run ends.
+    open: async () => {
+      const client = await connectPostgres(target.endpoint, { applicationName: "chant receipts" });
+      return { store: storeOver(client), close: () => client.end() };
+    },
+  };
+}
+
+/**
+ * The receipt store on the environment's database server. Nothing is read or
+ * connected until the first read or write; the environment, the server and
+ * the identity are resolved then, once.
+ */
+export function sqlReceiptStore(options: SqlReceiptStoreOptions = {}): SqlServerReceiptStore {
+  let resolved: Promise<Resolved> | undefined;
+  const resolvedOnce = () => (resolved ??= resolve(options));
+  const using = async <T>(fn: (store: SqlReceiptStore & { ensure?: () => Promise<void> }, r: Resolved) => Promise<T>): Promise<T> => {
+    const r = await resolvedOnce();
+    const { store, close } = await r.open();
+    try {
+      return await fn(store, r);
+    } finally {
+      await close();
+    }
+  };
+  return {
+    read: (receipt) => using((s) => s.read(receipt)),
+    readAll: (prefix) => using((s) => s.readAll(prefix)),
+    async write(receipt, expectation) {
+      await using(async (s) => {
+        // Made on the first write when it is not there. A caller writing
+        // inside its own transaction calls ensure() before opening it, so a
+        // rolled-back batch does not take the table with it.
+        await s.ensure?.();
+        await s.write(receipt, expectation);
+      });
+    },
+    ensure: () => using(async (s) => s.ensure?.()),
+    async location() {
+      const r = await resolvedOnce();
+      const { receiptAddress } = await import("./core/receipts");
+      return { dialect: r.dialect, table: r.table, source: r.source, address: receiptAddress(r.identity, "<effect>") };
+    },
+  };
+}
+
+/**
+ * The `receiptRead`, `receiptWrite` and `receiptStaleness` activities over
+ * {@link sqlReceiptStore}: what the op activities module exports, and what a
+ * caller running an Op of `effect()` steps with its own activity map binds.
+ */
+export function sqlReceiptActivities(options: SqlReceiptStoreOptions = {}, activityOptions?: ReceiptActivityOptions): ReceiptActivities {
+  return receiptActivities(sqlReceiptStore(options), activityOptions);
+}

package/src/skills/chant-sql-rebuild.md

@@ -28,6 +28,10 @@ export const { op } = ClickHouseRebuildOp({
 
 Options: `name`, `env`, `table`, `dualWrite` (required); `output` (default `dist/schema.json`), `path`, `build` (default true runs `chant build` first), `retain` (default `7d`), `gate` (the swap gate: `gate`, `timeout`, `description`, `approval`), `dropGate`, `writesGate` (app mode), `backfillTimeout` (default `6h`), `mutationTimeout` (default `10m`), `replicaTimeout` (default `2m`, Replicated databases only), `stack`, `ownershipEnv`.
 
+On a `cluster:<name>` topology with more than one shard, the backfill and the verification work per shard through the profile's server, with a receipt per shard and partition; each shard needs its own `{shard}` macro.
+
+Run as one step of a change approved elsewhere: `gates: "outer"` leaves out the swap gate, the drop gate and the Drop phase (verification and the swap's own comparison still run; the old table is kept), and `onFailure: "keep"` leaves out onFailure, so a failed run's next run resumes the backfill from its receipts. Set them rather than editing the Op's phases.
+
 Pick the dual-write mode:
 
 - `{ mode: "materialized-view", cutoverColumn: "ts", cutoverDelay?: "1m" }` when the table has a time column rows arrive in order of. Writes keep flowing: a materialized view sends rows at or after the cut-over to the new table, the backfill copies the rows before it.
@@ -51,7 +55,7 @@ The swap gate is bound to a digest of the plan and the verification (row counts
 ## When something goes wrong
 
 - A run stopped during the backfill (Ctrl-C, a killed job) is not a failure. The next run resumes from the receipts in `chant_receipts.receipts` on the same server (in `<db>.__chant_receipts` on a Replicated database), and a partition whose copy was cut off before its receipt is cleared and copied again, so no row is copied twice.
-- A failed step (a failed copy after its retries, a verification mismatch, a refused declaration) runs onFailure: the new table `<t>__chant_new` and the dual-write view `<t>__chant_dual` are dropped, and the next run starts from a new table. The original table is untouched until the swap.
+- A failed step (a failed copy after its retries, a verification mismatch, a refused declaration) runs onFailure: the new table `<t>__chant_new` and the dual-write view `<t>__chant_dual` are dropped, and the next run starts from a new table. With `onFailure: "keep"` they stay, and the next run resumes; drop them by hand to start over. The original table is untouched until the swap.
 - A verification mismatch in materialized-view mode usually means rows arrived with a time before the cut-over later than `cutoverDelay`. Raise the delay and run again.
 - The Op refuses a database that is neither Atomic nor Replicated (`EXCHANGE TABLES` needs one of them), a table whose change is not a rebuild (use the applier), and an object already under one of its working names that is not its own. In a Replicated database it also refuses a table, or a declaration, whose engine is not `Replicated*MergeTree`: a plain MergeTree keeps different rows on each replica and one replica's copy would lose the others'.