@intentius/sql-yodeler 0.0.0-stage → 0.1.0

package/dist/replay/report.d.ts

@@ -0,0 +1,18 @@
+/**
+ * The replay check's result for people (text) and for tools (JSON), and its
+ * exit code.
+ */
+import { type ReplayChange, type ReplayResult } from "./replay.js";
+/** 0 when every migration replays to its recorded schema; 1 for a difference or a statement the server refused; 2 for a step the replay cannot run. */
+export declare function replayExitCode(result: ReplayResult): number;
+/** One difference: the object, the field, and the replay's value against the record's. */
+export declare function describeChange(c: ReplayChange): string;
+/** The failure in one paragraph, naming the first migration that fails; `undefined` when the replay matched. For `yodel lint`. */
+export declare function replayFailure(result: ReplayResult): {
+    migration: string;
+    message: string;
+} | undefined;
+/** The result as text. */
+export declare function renderReplay(result: ReplayResult): string;
+/** The result as JSON. */
+export declare function replayJson(result: ReplayResult): Record<string, unknown>;

package/dist/replay/report.js

@@ -0,0 +1,72 @@
+/**
+ * The replay check's result for people (text) and for tools (JSON), and its
+ * exit code.
+ */
+import { MIGRATIONS_DIR, MIGRATION_JSON, MIGRATION_SQL } from "../migrations/index.js";
+import { UNREPLAYABLE } from "./replay.js";
+/** 0 when every migration replays to its recorded schema; 1 for a difference or a statement the server refused; 2 for a step the replay cannot run. */
+export function replayExitCode(result) {
+    if (result.outcome === "matches")
+        return 0;
+    if (result.outcome === "unreplayable")
+        return UNREPLAYABLE;
+    return 1;
+}
+const shown = (v) => (v === undefined ? "(absent)" : v);
+/** One difference: the object, the field, and the replay's value against the record's. */
+export function describeChange(c) {
+    return `${c.object}: ${c.field}: replay has ${shown(c.before)}, ${MIGRATION_JSON} records ${shown(c.after)} (${c.rule})`;
+}
+/** The failure in one paragraph, naming the first migration that fails; `undefined` when the replay matched. For `yodel lint`. */
+export function replayFailure(result) {
+    const before = result.matched.length > 0 ? ` The ${String(result.matched.length)} migration${result.matched.length === 1 ? "" : "s"} before it replayed to ${result.matched.length === 1 ? "its" : "their"} recorded schema${result.matched.length === 1 ? "" : "s"}.` : "";
+    switch (result.outcome) {
+        case "matches":
+            return undefined;
+        case "differs": {
+            const m = result.migration;
+            const n = result.changes.length;
+            const skipped = m.skipped.length > 0 ? `\nSteps not replayed: ${m.skipped.join("; ")}.` : "";
+            return {
+                migration: m.id,
+                message: `${m.id}: replaying the history through it does not give the schema it records (${MIGRATIONS_DIR}/${m.id}/${MIGRATION_JSON}); ${String(n)} difference${n === 1 ? "" : "s"}:\n` +
+                    result.changes.map((c) => `  ${describeChange(c)}`).join("\n") +
+                    `\n${before.trim() ? `${before.trim()}\n` : ""}` +
+                    `Its statements (the steps in ${MIGRATION_JSON}, which ${MIGRATION_SQL} shows) and its recorded schema disagree: make the statements produce the schema, or record the schema they produce, then yodel repair the migration where it is already applied.${skipped}`,
+            };
+        }
+        case "failed":
+            return {
+                migration: result.migration,
+                message: `${result.migration}: statement ${String(result.step)} failed in the replay: ${result.error}.${before} The history does not apply to an empty database as written.`,
+            };
+        case "unreplayable":
+            return {
+                migration: result.migration,
+                message: `${result.migration}: step ${String(result.step)} cannot be replayed: ${result.reason}.${before} The migrations from it on are not checked.`,
+            };
+    }
+}
+/** The result as text. */
+export function renderReplay(result) {
+    const lines = [`Replay of ${String(result.total)} migration${result.total === 1 ? "" : "s"} (${result.dialect}) into ${result.where}`];
+    if (result.scope.length > 0)
+        lines.push(`  ${result.dialect === "postgres" ? "schemas" : "databases"}: ${result.scope.join(", ")}${result.reset ? " (emptied first)" : ""}${result.kept ? " (left in place)" : ""}`);
+    for (const m of result.matched) {
+        const extra = [...m.ran.map((r) => `ran ${r}`), ...m.skipped.map((s) => `skipped ${s}`)];
+        lines.push(`  ok      ${m.id} (${String(m.statements)} statement${m.statements === 1 ? "" : "s"}${extra.length > 0 ? `; ${extra.join("; ")}` : ""})`);
+    }
+    const failure = replayFailure(result);
+    if (!failure) {
+        lines.push(result.total === 0 ? "No migrations to replay." : "Every migration replays to its recorded schema.");
+        return `${lines.join("\n")}\n`;
+    }
+    lines.push(`  ${result.outcome.padEnd(7)} ${failure.migration}`, "", failure.message);
+    return `${lines.join("\n")}\n`;
+}
+/** The result as JSON. */
+export function replayJson(result) {
+    const failure = replayFailure(result);
+    return { ...result, ok: result.outcome === "matches", ...(failure ? { message: failure.message } : {}) };
+}
+//# sourceMappingURL=report.js.map

package/dist/replay/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/replay/report.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvF,OAAO,EAAE,YAAY,EAAwC,MAAM,aAAa,CAAC;AAEjF,uJAAuJ;AACvJ,MAAM,UAAU,cAAc,CAAC,MAAoB;IACjD,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC;IAC3C,IAAI,MAAM,CAAC,OAAO,KAAK,cAAc;QAAE,OAAO,YAAY,CAAC;IAC3D,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,KAAK,GAAG,CAAC,CAAqB,EAAU,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEpF,0FAA0F;AAC1F,MAAM,UAAU,cAAc,CAAC,CAAe;IAC5C,OAAO,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,KAAK,gBAAgB,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,cAAc,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC;AAC3H,CAAC;AAED,kIAAkI;AAClI,MAAM,UAAU,aAAa,CAAC,MAAoB;IAChD,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,0BAA0B,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,mBAAmB,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5Q,QAAQ,MAAM,CAAC,OAAO,EAAE,CAAC;QACvB,KAAK,SAAS;YACZ,OAAO,SAAS,CAAC;QACnB,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC;YAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;YAChC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7F,OAAO;gBACL,SAAS,EAAE,CAAC,CAAC,EAAE;gBACf,OAAO,EACL,GAAG,CAAC,CAAC,EAAE,2EAA2E,cAAc,IAAI,CAAC,CAAC,EAAE,IAAI,cAAc,MAAM,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK;oBAC9K,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;oBAC9D,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;oBAChD,gCAAgC,cAAc,WAAW,aAAa,oLAAoL,OAAO,EAAE;aACtQ,CAAC;QACJ,CAAC;QACD,KAAK,QAAQ;YACX,OAAO;gBACL,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,OAAO,EAAE,GAAG,MAAM,CAAC,SAAS,eAAe,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,MAAM,CAAC,KAAK,IAAI,MAAM,8DAA8D;aAC7K,CAAC;QACJ,KAAK,cAAc;YACjB,OAAO;gBACL,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,OAAO,EAAE,GAAG,MAAM,CAAC,SAAS,UAAU,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,MAAM,CAAC,MAAM,IAAI,MAAM,6CAA6C;aACtJ,CAAC;IACN,CAAC;AACH,CAAC;AAED,0BAA0B;AAC1B,MAAM,UAAU,YAAY,CAAC,MAAoB;IAC/C,MAAM,KAAK,GAAG,CAAC,aAAa,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,MAAM,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,MAAM,CAAC,OAAO,UAAU,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IACvI,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,KAAK,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACrN,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;QACzF,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACxJ,CAAC;IACD,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACtC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,iDAAiD,CAAC,CAAC;QAChH,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;IACjC,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,SAAS,EAAE,EAAE,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IACtF,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AACjC,CAAC;AAED,0BAA0B;AAC1B,MAAM,UAAU,UAAU,CAAC,MAAoB;IAC7C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACtC,OAAO,EAAE,GAAG,MAAM,EAAE,EAAE,EAAE,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AAC3G,CAAC"}

package/dist/steps/backfill.d.ts

@@ -0,0 +1,128 @@
+/**
+ * A backfill step: a data migration the author writes, run inside the
+ * migration between the statements around it, resumable from receipts.
+ *
+ * The step names a module in the migration's directory (`backfill.ts` by
+ * default) that exports a chant Op as `op` (or as its default export). Each
+ * step of its phases is an `effect()` step: one batch of the backfill, with
+ * its own receipt. The nested steps run SQL on the environment's server with
+ * the `yodelSql` activity yodel provides, or any activity chant has:
+ *
+ *     import { EffectReceipt } from "@intentius/chant";
+ *     import { Op, activity, effect, phase } from "@intentius/chant/op";
+ *
+ *     const months = ["202601", "202602", "202603"];
+ *     const batch = (m: string) =>
+ *       effect(EffectReceipt(`country-${m}`, { effect: `country/${m}`, flavor: "existence" }), [
+ *         activity("yodelSql", { sql: `ALTER TABLE shop.orders UPDATE country = ... WHERE toYYYYMM(ts) = ${m}` }, "atMostOnce"),
+ *       ]);
+ *
+ *     export const op = Op({ name: "backfill-country", overview: "...", phases: [phase("Backfill", months.map(batch))] });
+ *
+ * How a run goes is chant's `effect()` cycle, run by chant's local executor:
+ * for each batch, read its receipt; skip the batch when the receipt is there
+ * (an earlier run finished it); otherwise run its steps and write the
+ * receipt last, only when they all succeeded. A run that fails or is killed
+ * part way leaves the finished batches' receipts, so the next `yodel apply`
+ * resumes at the first batch without one and never redoes a finished one.
+ * A batch that was running when it stopped runs again: make each batch safe
+ * to repeat (an idempotent `UPDATE`, or a delete of its rows before the
+ * insert), as the rebuild's backfill does per partition.
+ *
+ * The receipts are rows on the environment's server, addressed
+ * `yodel/<env>/<migration id>/<effect>`, so the same effect name in two
+ * migrations is two receipts:
+ *
+ * - ClickHouse: chant's receipts table (`chant_receipts.receipts`, the one
+ *   `ClickHouseRebuildOp` uses).
+ * - Postgres: `<history schema>.__chant_receipts`, chant's Postgres receipt
+ *   table (the one `PostgresMigrationOp` keeps in the migrated table's
+ *   schema), in the schema yodel's history is in. Each batch runs in one
+ *   transaction on the apply's connection, and its receipt is written last
+ *   inside it: the batch's statements and its receipt commit together or
+ *   not at all, so a run killed anywhere leaves no batch half done, and a
+ *   resumed backfill runs no batch twice. A batch's statements must be ones
+ *   Postgres runs in a transaction block (no `CREATE INDEX CONCURRENTLY`,
+ *   no `VACUUM`). The environment's `lockTimeoutMs` and `scanTimeoutMs` are
+ *   the batch's `lock_timeout` and `statement_timeout`, and `yodelSql`'s
+ *   `settings` are set for the batch's transaction (`set_config(..., true)`).
+ *
+ * The module is one of the migration's own files: it counts in the
+ * migration's checksum, so the plan digest an approval binds covers it, and
+ * an edit after it ran is a checksum mismatch. It is imported from the
+ * migration's directory, so it imports packages only (no relative imports).
+ * Its Op has no gates (the migration's gate approves it).
+ */
+import type { OpConfig } from "@intentius/chant/op/types";
+import type { SqlDialect } from "@intentius/chant-lexicon-sql/dialects";
+import type { ReceiptStore } from "@intentius/chant/op/receipt-store";
+import type { StepRecord } from "@intentius/chant/op/local-executor";
+import type { ClickHousePort } from "../history/index.js";
+import type { PgClient } from "../history/postgres/index.js";
+import type { PgTimeouts } from "../versioned/postgres/executor.js";
+import { type RunOpOptions } from "./run.js";
+import { type BackfillStep } from "./kinds.js";
+/** The activity a backfill's steps run SQL with: `activity("yodelSql", { sql, settings? })`. */
+export declare const SQL_ACTIVITY = "yodelSql";
+/** The step `yodel new --backfill` adds to a migration. */
+export declare function backfillStep(migrationName: string, file?: string): BackfillStep;
+/** Text only the template holds: a backfill still carrying it has not been written. */
+export declare const TEMPLATE_MARK = "write the backfill of";
+/** What `yodel new --backfill <file>` writes at `<file>` when there is none yet. */
+export declare function backfillTemplate(name: string, dialect?: SqlDialect): string;
+/** The Op config a module exports as `op` or as its default export. */
+export declare function exportedOp(mod: Record<string, unknown>): OpConfig | undefined;
+/** What is wrong with a backfill's Op, if anything: every phase step is an effect() step, and there are no gates. */
+export declare function backfillProblems(config: OpConfig): string[];
+/** Imports a backfill module and returns its Op, or throws naming what is wrong. `text` busts the import cache when the file changed. */
+export declare function loadBackfill(path: string, text?: string): Promise<OpConfig>;
+export interface BackfillSummary {
+    op: "backfill";
+    file: string;
+    /** The Op's name. */
+    name: string;
+    /** Batches (effect steps) the Op has. */
+    effects: number;
+    /** Batches skipped: their receipt was there (an earlier run finished them). */
+    skipped: number;
+    /** Batches run, and their receipt written, in this run. */
+    ran: number;
+}
+/** A receipt store whose every effect is under `prefix`, so receipts of different migrations never meet. */
+export declare function prefixedStore(store: ReceiptStore, prefix: string): ReceiptStore;
+/** The receipt store on a ClickHouse server: chant's, in `chant_receipts.receipts`. */
+export declare function serverReceipts(endpoint: {
+    url: string;
+    user?: string;
+    password?: string;
+}, environment: string): Promise<ReceiptStore>;
+/**
+ * The receipt store on a Postgres server: chant's, in `<schema>.__chant_receipts`,
+ * over `client` (so a receipt written inside a batch's transaction commits
+ * with it). Creates the table when it is not there.
+ */
+export declare function postgresServerReceipts(client: PgClient, schema: string, environment: string): Promise<ReceiptStore>;
+/** A backfill on Postgres: the connection its batches run on, and the environment's timeouts. */
+export interface PgBackfillServer {
+    client: PgClient;
+    timeouts: Pick<PgTimeouts, "lockTimeoutMs" | "scanTimeoutMs">;
+}
+export interface RunBackfillOptions extends Omit<RunOpOptions, "activities" | "cwd"> {
+    projectDir: string;
+    migrationId: string;
+    /** The migration's directory. */
+    dir: string;
+    /** The module's text as the migration read it (and checksummed). */
+    text?: string | undefined;
+    /** The ClickHouse server the SQL runs on; or `postgres`. */
+    port?: ClickHousePort | undefined;
+    /** The Postgres server the SQL runs on, a transaction per batch; or `port`. */
+    postgres?: PgBackfillServer | undefined;
+    receipts: ReceiptStore;
+    log?: (line: string) => void;
+}
+/** Runs a backfill step to the end, counting batches as they skip or finish. */
+export declare function runBackfill(file: string, run: RunBackfillOptions): Promise<{
+    summary: BackfillSummary;
+    records: StepRecord[];
+}>;

package/dist/steps/backfill.js

@@ -0,0 +1,307 @@
+/**
+ * A backfill step: a data migration the author writes, run inside the
+ * migration between the statements around it, resumable from receipts.
+ *
+ * The step names a module in the migration's directory (`backfill.ts` by
+ * default) that exports a chant Op as `op` (or as its default export). Each
+ * step of its phases is an `effect()` step: one batch of the backfill, with
+ * its own receipt. The nested steps run SQL on the environment's server with
+ * the `yodelSql` activity yodel provides, or any activity chant has:
+ *
+ *     import { EffectReceipt } from "@intentius/chant";
+ *     import { Op, activity, effect, phase } from "@intentius/chant/op";
+ *
+ *     const months = ["202601", "202602", "202603"];
+ *     const batch = (m: string) =>
+ *       effect(EffectReceipt(`country-${m}`, { effect: `country/${m}`, flavor: "existence" }), [
+ *         activity("yodelSql", { sql: `ALTER TABLE shop.orders UPDATE country = ... WHERE toYYYYMM(ts) = ${m}` }, "atMostOnce"),
+ *       ]);
+ *
+ *     export const op = Op({ name: "backfill-country", overview: "...", phases: [phase("Backfill", months.map(batch))] });
+ *
+ * How a run goes is chant's `effect()` cycle, run by chant's local executor:
+ * for each batch, read its receipt; skip the batch when the receipt is there
+ * (an earlier run finished it); otherwise run its steps and write the
+ * receipt last, only when they all succeeded. A run that fails or is killed
+ * part way leaves the finished batches' receipts, so the next `yodel apply`
+ * resumes at the first batch without one and never redoes a finished one.
+ * A batch that was running when it stopped runs again: make each batch safe
+ * to repeat (an idempotent `UPDATE`, or a delete of its rows before the
+ * insert), as the rebuild's backfill does per partition.
+ *
+ * The receipts are rows on the environment's server, addressed
+ * `yodel/<env>/<migration id>/<effect>`, so the same effect name in two
+ * migrations is two receipts:
+ *
+ * - ClickHouse: chant's receipts table (`chant_receipts.receipts`, the one
+ *   `ClickHouseRebuildOp` uses).
+ * - Postgres: `<history schema>.__chant_receipts`, chant's Postgres receipt
+ *   table (the one `PostgresMigrationOp` keeps in the migrated table's
+ *   schema), in the schema yodel's history is in. Each batch runs in one
+ *   transaction on the apply's connection, and its receipt is written last
+ *   inside it: the batch's statements and its receipt commit together or
+ *   not at all, so a run killed anywhere leaves no batch half done, and a
+ *   resumed backfill runs no batch twice. A batch's statements must be ones
+ *   Postgres runs in a transaction block (no `CREATE INDEX CONCURRENTLY`,
+ *   no `VACUUM`). The environment's `lockTimeoutMs` and `scanTimeoutMs` are
+ *   the batch's `lock_timeout` and `statement_timeout`, and `yodelSql`'s
+ *   `settings` are set for the batch's transaction (`set_config(..., true)`).
+ *
+ * The module is one of the migration's own files: it counts in the
+ * migration's checksum, so the plan digest an approval binds covers it, and
+ * an edit after it ran is a checksum mismatch. It is imported from the
+ * migration's directory, so it imports packages only (no relative imports).
+ * Its Op has no gates (the migration's gate approves it).
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+import { YodelError } from "../project.js";
+import { gatesOf, runOp, StepOpFailed } from "./run.js";
+import { BACKFILL_FILE, BACKFILL_OP } from "./kinds.js";
+/** The activity a backfill's steps run SQL with: `activity("yodelSql", { sql, settings? })`. */
+export const SQL_ACTIVITY = "yodelSql";
+/** The step `yodel new --backfill` adds to a migration. */
+export function backfillStep(migrationName, file = BACKFILL_FILE) {
+    return {
+        kind: "op",
+        object: "backfill",
+        type: "Yodel::Backfill",
+        name: migrationName,
+        op: BACKFILL_OP,
+        importPath: `./${file}`,
+        options: { file },
+        declaration: `the Op exported by ${file} in this migration's directory`,
+        changes: [],
+        detail: `a data backfill: the effect() steps of the Op in ${file}, each a batch with its own receipt; yodel apply runs it after the statements before it and resumes it from its receipts`,
+    };
+}
+/** Text only the template holds: a backfill still carrying it has not been written. */
+export const TEMPLATE_MARK = "write the backfill of";
+/** What `yodel new --backfill <file>` writes at `<file>` when there is none yet. */
+export function backfillTemplate(name, dialect = "clickhouse") {
+    const todo = dialect === "postgres"
+        ? `DO $$ BEGIN RAISE EXCEPTION 'write the backfill of ${name} for batch \${b}'; END $$`
+        : `SELECT throwIf(1, 'write the backfill of ${name} for batch \${b}')`;
+    return `/**
+ * The backfill of migration ${name}. yodel apply runs the Op this file
+ * exports, after the migration's statements before the step and before
+ * those after it. Each effect() is one batch with its own receipt: a rerun
+ * after a failure skips the batches that finished. ${dialect === "postgres"
+        ? `Each batch is one transaction
+ * with its receipt, so a batch that was running when a run stopped left
+ * nothing and runs again whole.`
+        : `Make each batch safe to
+ * run twice (the one running when a run stopped runs again).`}
+ *
+ * This file is copied into the migration's directory and counts in its
+ * checksum. Import packages only, no relative paths. No gates: the
+ * migration's own gate approves it.
+ */
+import { EffectReceipt } from "@intentius/chant";
+import { Op, activity, effect, phase } from "@intentius/chant/op";
+
+// The batches: months, id ranges, partitions.
+const batches: string[] = ["202601", "202602"];
+
+const batch = (b: string) =>
+  effect(EffectReceipt(\`${name}-\${b}\`, { effect: \`${name}/\${b}\`, flavor: "existence" }), [
+    // TODO: the SQL for one batch.
+    activity("${SQL_ACTIVITY}", { sql: \`${todo}\` }, "atMostOnce"),
+  ]);
+
+export const op = Op({
+  name: "backfill-${name}",
+  overview: "Backfill for migration ${name}",
+  phases: [phase("Backfill", batches.map(batch))],
+});
+`;
+}
+const isRecord = (v) => typeof v === "object" && v !== null;
+/** The Op config a module exports as `op` or as its default export. */
+export function exportedOp(mod) {
+    for (const value of [mod.op, mod.default]) {
+        const props = isRecord(value) ? value.props : undefined;
+        if (isRecord(props) && typeof props.name === "string" && Array.isArray(props.phases))
+            return props;
+    }
+    return undefined;
+}
+/** What is wrong with a backfill's Op, if anything: every phase step is an effect() step, and there are no gates. */
+export function backfillProblems(config) {
+    const problems = [];
+    const steps = config.phases.flatMap((p) => p.steps.map((s) => ({ phase: p.name, s })));
+    if (steps.length === 0)
+        problems.push("its Op has no steps");
+    for (const { phase, s } of steps) {
+        if (s.kind !== "effect")
+            problems.push(`phase "${phase}" has a ${s.kind === "gate" ? `gate (${s.gate ?? "unnamed"})` : `${s.fn} step`} outside an effect(): every step of a backfill is an effect() with its receipt, so a rerun resumes`);
+    }
+    const gates = gatesOf(config);
+    if (gates.length > 0)
+        problems.push(`it has a gate (${gates.join(", ")}); the migration's own gate approves the backfill`);
+    return problems;
+}
+/** Imports a backfill module and returns its Op, or throws naming what is wrong. `text` busts the import cache when the file changed. */
+export async function loadBackfill(path, text) {
+    if (!existsSync(path))
+        throw new YodelError(`${path}: no such file; the backfill step runs the Op it exports`);
+    const body = text ?? readFileSync(path, "utf8");
+    const version = createHash("sha256").update(body).digest("hex").slice(0, 16);
+    let mod;
+    try {
+        mod = (await import(`${pathToFileURL(path).href}?yodel=${version}`));
+    }
+    catch (e) {
+        throw new YodelError(`${path}: could not load it: ${e instanceof Error ? e.message : String(e)}`);
+    }
+    const config = exportedOp(mod);
+    if (!config)
+        throw new YodelError(`${path} exports no Op (export const op = Op({ ... }), or a default export)`);
+    const problems = backfillProblems(config);
+    if (problems.length > 0)
+        throw new YodelError(`${path}: ${problems.join("; ")}`);
+    return config;
+}
+/** A receipt store whose every effect is under `prefix`, so receipts of different migrations never meet. */
+export function prefixedStore(store, prefix) {
+    return {
+        read: (r) => store.read({ ...r, effect: `${prefix}${r.effect}` }),
+        write: (r, expectation) => store.write({ ...r, effect: `${prefix}${r.effect}` }, expectation),
+    };
+}
+/** The receipt store on a ClickHouse server: chant's, in `chant_receipts.receipts`. */
+export async function serverReceipts(endpoint, environment) {
+    const { clickhouseReceiptStore } = await import("@intentius/chant-lexicon-sql/clickhouse/rebuild/receipts");
+    return clickhouseReceiptStore(endpoint, { stack: "yodel", env: environment });
+}
+/**
+ * The receipt store on a Postgres server: chant's, in `<schema>.__chant_receipts`,
+ * over `client` (so a receipt written inside a batch's transaction commits
+ * with it). Creates the table when it is not there.
+ */
+export async function postgresServerReceipts(client, schema, environment) {
+    const { postgresReceiptStore } = await import("@intentius/chant-lexicon-sql/postgres/migrate/receipts");
+    // The store never closes the apply's connection.
+    const store = postgresReceiptStore({ query: (sql, params) => client.query(sql, params), end: () => Promise.resolve() }, schema, { stack: "yodel", env: environment });
+    await store.ensure();
+    return store;
+}
+function clickhouseBatches(port) {
+    return {
+        next: () => Promise.resolve(),
+        sql: async (sql, settings) => (await port.query(sql, settings ? { settings } : {})).length,
+        done: () => Promise.resolve(),
+        abandon: () => Promise.resolve(),
+    };
+}
+/** Each batch in one transaction, begun at its first statement and committed after its receipt. */
+function postgresBatches(server) {
+    const { client, timeouts } = server;
+    let open = false;
+    /** Why this batch's transaction was rolled back: its later statements (an activity's retry) do not run without the earlier ones. */
+    let lost;
+    const rollback = async () => {
+        if (!open)
+            return;
+        open = false;
+        await client.query("ROLLBACK").catch(() => undefined);
+    };
+    return {
+        async next() {
+            await rollback();
+            lost = undefined;
+        },
+        async sql(sql, settings) {
+            if (lost !== undefined)
+                throw new Error(`this batch's transaction was rolled back (${lost}); the batch runs again whole on the next yodel apply`);
+            try {
+                if (!open) {
+                    await client.query("BEGIN");
+                    open = true;
+                    await client.query(`SET LOCAL lock_timeout = '${String(Math.max(0, Math.floor(timeouts.lockTimeoutMs)))}ms'`);
+                    await client.query(`SET LOCAL statement_timeout = '${String(Math.max(0, Math.floor(timeouts.scanTimeoutMs)))}ms'`);
+                }
+                for (const [name, value] of Object.entries(settings ?? {}))
+                    await client.query("SELECT pg_catalog.set_config($1, $2, true)", [name, value]);
+                return (await client.query(sql)).length;
+            }
+            catch (e) {
+                lost = e instanceof Error ? e.message : String(e);
+                await rollback();
+                throw e;
+            }
+        },
+        async done() {
+            if (!open)
+                return;
+            open = false;
+            await client.query("COMMIT");
+        },
+        abandon: rollback,
+    };
+}
+/** Runs a backfill step to the end, counting batches as they skip or finish. */
+export async function runBackfill(file, run) {
+    const config = await loadBackfill(join(run.dir, file), run.text);
+    const summary = { op: "backfill", file, name: config.name, effects: config.phases.reduce((n, p) => n + p.steps.length, 0), skipped: 0, ran: 0 };
+    const session = run.postgres ? postgresBatches(run.postgres) : run.port ? clickhouseBatches(run.port) : undefined;
+    if (!session)
+        throw new YodelError("a backfill step needs the environment's server to run its SQL on, and has none");
+    const { receiptActivities } = await import("@intentius/chant/op/receipt-store");
+    const store = receiptActivities(prefixedStore(run.receipts, `${run.migrationId}/`));
+    const activities = {
+        receiptRead: async (args) => {
+            // A new batch: nothing of an earlier one is left open.
+            await session.next();
+            const r = await store.receiptRead(args);
+            if (r.applied)
+                summary.skipped++;
+            return r;
+        },
+        receiptWrite: async (args) => {
+            let r;
+            try {
+                r = await store.receiptWrite(args);
+                await session.done();
+            }
+            catch (e) {
+                await session.abandon();
+                throw e;
+            }
+            summary.ran++;
+            return r;
+        },
+        receiptStaleness: (args) => store.receiptStaleness(args),
+        [SQL_ACTIVITY]: async (args) => {
+            const sql = args.sql;
+            if (typeof sql !== "string" || sql.trim() === "")
+                throw new Error(`${SQL_ACTIVITY}: needs { sql: "<statement>" }`);
+            const settings = isRecord(args.settings) ? Object.fromEntries(Object.entries(args.settings).map(([k, v]) => [k, String(v)])) : undefined;
+            run.log?.(`    ${sql.split("\n")[0] ?? ""}`);
+            return { rows: await session.sql(sql, settings) };
+        },
+    };
+    try {
+        const records = await runOp(config, {
+            cwd: run.projectDir,
+            environment: run.environment,
+            activities,
+            ...(run.base ? { base: run.base } : {}),
+            ...(run.signal ? { signal: run.signal } : {}),
+            ...(run.onRecord ? { onRecord: run.onRecord } : {}),
+        });
+        return { summary, records };
+    }
+    catch (e) {
+        if (e instanceof StepOpFailed)
+            e.summary = summary;
+        throw e;
+    }
+    finally {
+        // A batch stopped part way (a failed step that was not ours, an abort) leaves nothing behind.
+        await session.abandon();
+    }
+}
+//# sourceMappingURL=backfill.js.map

package/dist/steps/backfill.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backfill.js","sourceRoot":"","sources":["../../src/steps/backfill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAMzC,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAI3C,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,EAAqB,MAAM,UAAU,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,WAAW,EAAqB,MAAM,YAAY,CAAC;AAE3E,gGAAgG;AAChG,MAAM,CAAC,MAAM,YAAY,GAAG,UAAU,CAAC;AAEvC,2DAA2D;AAC3D,MAAM,UAAU,YAAY,CAAC,aAAqB,EAAE,IAAI,GAAG,aAAa;IACtE,OAAO;QACL,IAAI,EAAE,IAAI;QACV,MAAM,EAAE,UAAU;QAClB,IAAI,EAAE,iBAAiB;QACvB,IAAI,EAAE,aAAa;QACnB,EAAE,EAAE,WAAW;QACf,UAAU,EAAE,KAAK,IAAI,EAAE;QACvB,OAAO,EAAE,EAAE,IAAI,EAAE;QACjB,WAAW,EAAE,sBAAsB,IAAI,gCAAgC;QACvE,OAAO,EAAE,EAAE;QACX,MAAM,EAAE,oDAAoD,IAAI,0HAA0H;KAC3L,CAAC;AACJ,CAAC;AAED,uFAAuF;AACvF,MAAM,CAAC,MAAM,aAAa,GAAG,uBAAuB,CAAC;AAErD,oFAAoF;AACpF,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,UAAsB,YAAY;IAC/E,MAAM,IAAI,GACR,OAAO,KAAK,UAAU;QACpB,CAAC,CAAC,sDAAsD,IAAI,2BAA2B;QACvF,CAAC,CAAC,4CAA4C,IAAI,oBAAoB,CAAC;IAC3E,OAAO;+BACsB,IAAI;;;sDAI/B,OAAO,KAAK,UAAU;QACpB,CAAC,CAAC;;iCAEyB;QAC3B,CAAC,CAAC;8DAEN;;;;;;;;;;;;;2BAayB,IAAI,yBAAyB,IAAI;;gBAE5C,YAAY,eAAe,IAAI;;;;oBAI3B,IAAI;sCACc,IAAI;;;CAGzC,CAAC;AACF,CAAC;AAED,MAAM,QAAQ,GAAG,CAAC,CAAU,EAAgC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,CAAC;AAEnG,uEAAuE;AACvE,MAAM,UAAU,UAAU,CAAC,GAA4B;IACrD,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QACxD,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;YAAE,OAAO,KAA4B,CAAC;IAC5H,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,qHAAqH;AACrH,MAAM,UAAU,gBAAgB,CAAC,MAAgB;IAC/C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC7D,KAAK,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;QACjC,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ;YAAE,QAAQ,CAAC,IAAI,CAAC,UAAU,KAAK,WAAW,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,oGAAoG,CAAC,CAAC;IAC7O,CAAC;IACD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,kBAAkB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,mDAAmD,CAAC,CAAC;IAC3H,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,yIAAyI;AACzI,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,IAAa;IAC5D,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,0DAA0D,CAAC,CAAC;IAC/G,MAAM,IAAI,GAAG,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC7E,IAAI,GAA4B,CAAC;IACjC,IAAI,CAAC;QACH,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,UAAU,OAAO,EAAE,CAAC,CAA4B,CAAC;IAClG,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,wBAAwB,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpG,CAAC;IACD,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,qEAAqE,CAAC,CAAC;IAChH,MAAM,QAAQ,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjF,OAAO,MAAM,CAAC;AAChB,CAAC;AAeD,4GAA4G;AAC5G,MAAM,UAAU,aAAa,CAAC,KAAmB,EAAE,MAAc;IAC/D,OAAO;QACL,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;QACjE,KAAK,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,EAAE,WAAW,CAAC;KAC9F,CAAC;AACJ,CAAC;AAED,uFAAuF;AACvF,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAA2D,EAAE,WAAmB;IACnH,MAAM,EAAE,sBAAsB,EAAE,GAAG,MAAM,MAAM,CAAC,0DAA0D,CAAC,CAAC;IAC5G,OAAO,sBAAsB,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,CAAC;AAChF,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,MAAgB,EAAE,MAAc,EAAE,WAAmB;IAChG,MAAM,EAAE,oBAAoB,EAAE,GAAG,MAAM,MAAM,CAAC,wDAAwD,CAAC,CAAC;IACxG,iDAAiD;IACjD,MAAM,KAAK,GAAG,oBAAoB,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,CAAC;IACtK,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;IACrB,OAAO,KAAK,CAAC;AACf,CAAC;AAgBD,SAAS,iBAAiB,CAAC,IAAoB;IAC7C,OAAO;QACL,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE;QAC7B,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM;QAC1F,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE;QAC7B,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE;KACjC,CAAC;AACJ,CAAC;AAED,mGAAmG;AACnG,SAAS,eAAe,CAAC,MAAwB;IAC/C,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;IACpC,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,oIAAoI;IACpI,IAAI,IAAwB,CAAC;IAC7B,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QAC1B,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,IAAI,GAAG,KAAK,CAAC;QACb,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACxD,CAAC,CAAC;IACF,OAAO;QACL,KAAK,CAAC,IAAI;YACR,MAAM,QAAQ,EAAE,CAAC;YACjB,IAAI,GAAG,SAAS,CAAC;QACnB,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ;YACrB,IAAI,IAAI,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,IAAI,uDAAuD,CAAC,CAAC;YAClJ,IAAI,CAAC;gBACH,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC5B,IAAI,GAAG,IAAI,CAAC;oBACZ,MAAM,MAAM,CAAC,KAAK,CAAC,6BAA6B,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;oBAC9G,MAAM,MAAM,CAAC,KAAK,CAAC,kCAAkC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBACrH,CAAC;gBACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;oBAAE,MAAM,MAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC5I,OAAO,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;YAC1C,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAClD,MAAM,QAAQ,EAAE,CAAC;gBACjB,MAAM,CAAC,CAAC;YACV,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI;YACR,IAAI,CAAC,IAAI;gBAAE,OAAO;YAClB,IAAI,GAAG,KAAK,CAAC;YACb,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,EAAE,QAAQ;KAClB,CAAC;AACJ,CAAC;AAiBD,gFAAgF;AAChF,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,GAAuB;IACrE,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IACjE,MAAM,OAAO,GAAoB,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;IACjK,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAClH,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,UAAU,CAAC,gFAAgF,CAAC,CAAC;IACrH,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,mCAAmC,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,iBAAiB,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;IACpF,MAAM,UAAU,GAA+B;QAC7C,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAC1B,uDAAuD;YACvD,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;YACrB,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,IAA0D,CAAC,CAAC;YAC9F,IAAI,CAAC,CAAC,OAAO;gBAAE,OAAO,CAAC,OAAO,EAAE,CAAC;YACjC,OAAO,CAAC,CAAC;QACX,CAAC;QACD,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAC3B,IAAI,CAAiD,CAAC;YACtD,IAAI,CAAC;gBACH,CAAC,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,IAA2D,CAAC,CAAC;gBAC1F,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;YACvB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;gBACxB,MAAM,CAAC,CAAC;YACV,CAAC;YACD,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,CAAC;QACX,CAAC;QACD,gBAAgB,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAA+D,CAAC;QACnH,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;YACrB,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,GAAG,YAAY,gCAAgC,CAAC,CAAC;YACnH,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAA4B,CAAC,CAAC,CAAC,SAAS,CAAC;YACrK,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC7C,OAAO,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;QACpD,CAAC;KACF,CAAC;IACF,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE;YAClC,GAAG,EAAE,GAAG,CAAC,UAAU;YACnB,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,UAAU;YACV,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7C,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpD,CAAC,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC9B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,YAAY;YAAE,CAAC,CAAC,OAAO,GAAG,OAAO,CAAC;QACnD,MAAM,CAAC,CAAC;IACV,CAAC;YAAS,CAAC;QACT,8FAA8F;QAC9F,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC"}

package/dist/steps/index.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Op steps inside a migration (#12): what `yodel apply` runs besides SQL.
+ *
+ *   planStep(step)            statement, rebuild, backfill, or refused (with why)
+ *   stepSha(step, files)      the hash a step's history rows record
+ *   runRebuild(...)           a ClickHouseRebuildOp step (./rebuild.ts)
+ *   runBackfill(...)          a backfill step, an Op of effect() batches (./backfill.ts)
+ *   runPgMigration(...)       a PostgresMigrationOp step (./postgres-migration.ts)
+ *   defaultStepRunner         what the versioned apply calls for an Op step
+ */
+import type { ReceiptStore } from "@intentius/chant/op/receipt-store";
+import type { ClickHouseEndpoint } from "@intentius/chant-lexicon-sql/clickhouse/http";
+import type { ClickHousePort } from "../history/index.js";
+import type { Migration } from "../migrations/index.js";
+import { type PgBackfillServer } from "./backfill.js";
+import type { StepPlan } from "./kinds.js";
+export * from "./kinds.js";
+export * from "./run.js";
+export * from "./rebuild.js";
+export * from "./backfill.js";
+export * from "./postgres-migration.js";
+/** What running one Op step needs. */
+export interface StepContext {
+    environment: string;
+    /** The project directory, which holds chant.config.ts. */
+    projectDir: string;
+    /** The ClickHouse server a backfill's SQL runs on; absent on Postgres. */
+    port?: ClickHousePort | undefined;
+    /** Postgres: the apply's connection a backfill's SQL runs on, its timeouts, and the schema its receipts are kept in (the history's). */
+    postgres?: (PgBackfillServer & {
+        receiptsSchema: string;
+    }) | undefined;
+    /** Postgres: the environment's lock timeout, for a PostgresMigrationOp step whose options set none. */
+    lockTimeoutMs?: number | undefined;
+    /** The server, for chant's receipt store; the port is enough for everything else. */
+    endpoint?: ClickHouseEndpoint | undefined;
+    migration: Migration;
+    index: number;
+    plan: Exclude<StepPlan, {
+        run: "statement";
+    } | {
+        run: "refused";
+    }>;
+    log: (line: string) => void;
+    /** A backfill's receipt store; default chant's on the server. */
+    receipts?: ReceiptStore | undefined;
+}
+/** Runs one Op step to the end and returns the summary its history row records. Throws `StepOpFailed` when it stops part way. */
+export type StepRunner = (ctx: StepContext) => Promise<{
+    summary: unknown;
+}>;
+export declare const defaultStepRunner: StepRunner;