@intentius/sql-yodeler 0.0.0-stage → 0.2.0

package/dist/steps/backfill.js

@@ -0,0 +1,307 @@
+/**
+ * A backfill step: a data migration the author writes, run inside the
+ * migration between the statements around it, resumable from receipts.
+ *
+ * The step names a module in the migration's directory (`backfill.ts` by
+ * default) that exports a chant Op as `op` (or as its default export). Each
+ * step of its phases is an `effect()` step: one batch of the backfill, with
+ * its own receipt. The nested steps run SQL on the environment's server with
+ * the `yodelSql` activity yodel provides, or any activity chant has:
+ *
+ *     import { EffectReceipt } from "@intentius/chant";
+ *     import { Op, activity, effect, phase } from "@intentius/chant/op";
+ *
+ *     const months = ["202601", "202602", "202603"];
+ *     const batch = (m: string) =>
+ *       effect(EffectReceipt(`country-${m}`, { effect: `country/${m}`, flavor: "existence" }), [
+ *         activity("yodelSql", { sql: `ALTER TABLE shop.orders UPDATE country = ... WHERE toYYYYMM(ts) = ${m}` }, "atMostOnce"),
+ *       ]);
+ *
+ *     export const op = Op({ name: "backfill-country", overview: "...", phases: [phase("Backfill", months.map(batch))] });
+ *
+ * How a run goes is chant's `effect()` cycle, run by chant's local executor:
+ * for each batch, read its receipt; skip the batch when the receipt is there
+ * (an earlier run finished it); otherwise run its steps and write the
+ * receipt last, only when they all succeeded. A run that fails or is killed
+ * part way leaves the finished batches' receipts, so the next `yodel apply`
+ * resumes at the first batch without one and never redoes a finished one.
+ * A batch that was running when it stopped runs again: make each batch safe
+ * to repeat (an idempotent `UPDATE`, or a delete of its rows before the
+ * insert), as the rebuild's backfill does per partition.
+ *
+ * The receipts are rows on the environment's server, kept by the sql
+ * lexicon's receipt store (`sqlReceiptStore`), addressed
+ * `yodel/<env>/<migration id>/<effect>`, so the same effect name in two
+ * migrations is two receipts:
+ *
+ * - ClickHouse: `chant_receipts.receipts` (the table `ClickHouseRebuildOp`
+ *   keeps its partition receipts in).
+ * - Postgres: `<history schema>.__chant_receipts`, in the schema yodel's
+ *   history is in, over the apply's connection. Each batch runs in one
+ *   transaction on that connection, and its receipt is written last
+ *   inside it: the batch's statements and its receipt commit together or
+ *   not at all, so a run killed anywhere leaves no batch half done, and a
+ *   resumed backfill runs no batch twice. A batch's statements must be ones
+ *   Postgres runs in a transaction block (no `CREATE INDEX CONCURRENTLY`,
+ *   no `VACUUM`). The environment's `lockTimeoutMs` and `scanTimeoutMs` are
+ *   the batch's `lock_timeout` and `statement_timeout`, and `yodelSql`'s
+ *   `settings` are set for the batch's transaction (`set_config(..., true)`).
+ *
+ * The module is one of the migration's own files: it counts in the
+ * migration's checksum, so the plan digest an approval binds covers it, and
+ * an edit after it ran is a checksum mismatch. It is imported from the
+ * migration's directory, so it imports packages only (no relative imports).
+ * Its Op has no gates (the migration's gate approves it).
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+import { YodelError } from "../project.js";
+import { gatesOf, runOp, StepOpFailed } from "./run.js";
+import { BACKFILL_FILE, BACKFILL_OP } from "./kinds.js";
+/** The activity a backfill's steps run SQL with: `activity("yodelSql", { sql, settings? })`. */
+export const SQL_ACTIVITY = "yodelSql";
+/** The step `yodel new --backfill` adds to a migration. */
+export function backfillStep(migrationName, file = BACKFILL_FILE) {
+    return {
+        kind: "op",
+        object: "backfill",
+        type: "Yodel::Backfill",
+        name: migrationName,
+        op: BACKFILL_OP,
+        importPath: `./${file}`,
+        options: { file },
+        declaration: `the Op exported by ${file} in this migration's directory`,
+        changes: [],
+        detail: `a data backfill: the effect() steps of the Op in ${file}, each a batch with its own receipt; yodel apply runs it after the statements before it and resumes it from its receipts`,
+    };
+}
+/** Text only the template holds: a backfill still carrying it has not been written. */
+export const TEMPLATE_MARK = "write the backfill of";
+/** What `yodel new --backfill <file>` writes at `<file>` when there is none yet. */
+export function backfillTemplate(name, dialect = "clickhouse") {
+    const todo = dialect === "postgres"
+        ? `DO $$ BEGIN RAISE EXCEPTION 'write the backfill of ${name} for batch \${b}'; END $$`
+        : `SELECT throwIf(1, 'write the backfill of ${name} for batch \${b}')`;
+    return `/**
+ * The backfill of migration ${name}. yodel apply runs the Op this file
+ * exports, after the migration's statements before the step and before
+ * those after it. Each effect() is one batch with its own receipt: a rerun
+ * after a failure skips the batches that finished. ${dialect === "postgres"
+        ? `Each batch is one transaction
+ * with its receipt, so a batch that was running when a run stopped left
+ * nothing and runs again whole.`
+        : `Make each batch safe to
+ * run twice (the one running when a run stopped runs again).`}
+ *
+ * This file is copied into the migration's directory and counts in its
+ * checksum. Import packages only, no relative paths. No gates: the
+ * migration's own gate approves it.
+ */
+import { EffectReceipt } from "@intentius/chant";
+import { Op, activity, effect, phase } from "@intentius/chant/op";
+
+// The batches: months, id ranges, partitions.
+const batches: string[] = ["202601", "202602"];
+
+const batch = (b: string) =>
+  effect(EffectReceipt(\`${name}-\${b}\`, { effect: \`${name}/\${b}\`, flavor: "existence" }), [
+    // TODO: the SQL for one batch.
+    activity("${SQL_ACTIVITY}", { sql: \`${todo}\` }, "atMostOnce"),
+  ]);
+
+export const op = Op({
+  name: "backfill-${name}",
+  overview: "Backfill for migration ${name}",
+  phases: [phase("Backfill", batches.map(batch))],
+});
+`;
+}
+const isRecord = (v) => typeof v === "object" && v !== null;
+/** The Op config a module exports as `op` or as its default export. */
+export function exportedOp(mod) {
+    for (const value of [mod.op, mod.default]) {
+        const props = isRecord(value) ? value.props : undefined;
+        if (isRecord(props) && typeof props.name === "string" && Array.isArray(props.phases))
+            return props;
+    }
+    return undefined;
+}
+/** What is wrong with a backfill's Op, if anything: every phase step is an effect() step, and there are no gates. */
+export function backfillProblems(config) {
+    const problems = [];
+    const steps = config.phases.flatMap((p) => p.steps.map((s) => ({ phase: p.name, s })));
+    if (steps.length === 0)
+        problems.push("its Op has no steps");
+    for (const { phase, s } of steps) {
+        if (s.kind !== "effect")
+            problems.push(`phase "${phase}" has a ${s.kind === "gate" ? `gate (${s.gate ?? "unnamed"})` : `${s.fn} step`} outside an effect(): every step of a backfill is an effect() with its receipt, so a rerun resumes`);
+    }
+    const gates = gatesOf(config);
+    if (gates.length > 0)
+        problems.push(`it has a gate (${gates.join(", ")}); the migration's own gate approves the backfill`);
+    return problems;
+}
+/** Imports a backfill module and returns its Op, or throws naming what is wrong. `text` busts the import cache when the file changed. */
+export async function loadBackfill(path, text) {
+    if (!existsSync(path))
+        throw new YodelError(`${path}: no such file; the backfill step runs the Op it exports`);
+    const body = text ?? readFileSync(path, "utf8");
+    const version = createHash("sha256").update(body).digest("hex").slice(0, 16);
+    let mod;
+    try {
+        mod = (await import(`${pathToFileURL(path).href}?yodel=${version}`));
+    }
+    catch (e) {
+        throw new YodelError(`${path}: could not load it: ${e instanceof Error ? e.message : String(e)}`);
+    }
+    const config = exportedOp(mod);
+    if (!config)
+        throw new YodelError(`${path} exports no Op (export const op = Op({ ... }), or a default export)`);
+    const problems = backfillProblems(config);
+    if (problems.length > 0)
+        throw new YodelError(`${path}: ${problems.join("; ")}`);
+    return config;
+}
+/** A receipt store whose every effect is under `prefix`, so receipts of different migrations never meet. */
+export function prefixedStore(store, prefix) {
+    return {
+        read: (r) => store.read({ ...r, effect: `${prefix}${r.effect}` }),
+        write: (r, expectation) => store.write({ ...r, effect: `${prefix}${r.effect}` }, expectation),
+    };
+}
+/**
+ * The sql lexicon's receipt store on the environment's server, under the
+ * stack `yodel`. On Postgres it writes over the apply's connection (so a
+ * receipt written inside a batch's transaction commits with it, and the
+ * store never closes it), and the table is made here, before any batch's
+ * transaction opens.
+ */
+export async function serverReceipts(server, environment) {
+    const { sqlReceiptStore } = await import("@intentius/chant-lexicon-sql/receipts");
+    const where = "clickhouse" in server
+        ? { clickhouse: { endpoint: server.clickhouse } }
+        : { postgres: { query: (sql, params) => server.postgres.query(sql, params), end: () => Promise.resolve() }, schema: server.schema };
+    const store = sqlReceiptStore({ environment, stack: "yodel", config: {}, ...where });
+    if ("postgres" in server)
+        await store.ensure();
+    return store;
+}
+function clickhouseBatches(port) {
+    return {
+        next: () => Promise.resolve(),
+        sql: async (sql, settings) => (await port.query(sql, settings ? { settings } : {})).length,
+        done: () => Promise.resolve(),
+        abandon: () => Promise.resolve(),
+    };
+}
+/** Each batch in one transaction, begun at its first statement and committed after its receipt. */
+function postgresBatches(server) {
+    const { client, timeouts } = server;
+    let open = false;
+    /** Why this batch's transaction was rolled back: its later statements (an activity's retry) do not run without the earlier ones. */
+    let lost;
+    const rollback = async () => {
+        if (!open)
+            return;
+        open = false;
+        await client.query("ROLLBACK").catch(() => undefined);
+    };
+    return {
+        async next() {
+            await rollback();
+            lost = undefined;
+        },
+        async sql(sql, settings) {
+            if (lost !== undefined)
+                throw new Error(`this batch's transaction was rolled back (${lost}); the batch runs again whole on the next yodel apply`);
+            try {
+                if (!open) {
+                    await client.query("BEGIN");
+                    open = true;
+                    await client.query(`SET LOCAL lock_timeout = '${String(Math.max(0, Math.floor(timeouts.lockTimeoutMs)))}ms'`);
+                    await client.query(`SET LOCAL statement_timeout = '${String(Math.max(0, Math.floor(timeouts.scanTimeoutMs)))}ms'`);
+                }
+                for (const [name, value] of Object.entries(settings ?? {}))
+                    await client.query("SELECT pg_catalog.set_config($1, $2, true)", [name, value]);
+                return (await client.query(sql)).length;
+            }
+            catch (e) {
+                lost = e instanceof Error ? e.message : String(e);
+                await rollback();
+                throw e;
+            }
+        },
+        async done() {
+            if (!open)
+                return;
+            open = false;
+            await client.query("COMMIT");
+        },
+        abandon: rollback,
+    };
+}
+/** Runs a backfill step to the end, counting batches as they skip or finish. */
+export async function runBackfill(file, run) {
+    const config = await loadBackfill(join(run.dir, file), run.text);
+    const summary = { op: "backfill", file, name: config.name, effects: config.phases.reduce((n, p) => n + p.steps.length, 0), skipped: 0, ran: 0 };
+    const session = run.postgres ? postgresBatches(run.postgres) : run.port ? clickhouseBatches(run.port) : undefined;
+    if (!session)
+        throw new YodelError("a backfill step needs the environment's server to run its SQL on, and has none");
+    const { receiptActivities } = await import("@intentius/chant/op/receipt-store");
+    const store = receiptActivities(prefixedStore(run.receipts, `${run.migrationId}/`));
+    const activities = {
+        receiptRead: async (args) => {
+            // A new batch: nothing of an earlier one is left open.
+            await session.next();
+            const r = await store.receiptRead(args);
+            if (r.applied)
+                summary.skipped++;
+            return r;
+        },
+        receiptWrite: async (args) => {
+            let r;
+            try {
+                r = await store.receiptWrite(args);
+                await session.done();
+            }
+            catch (e) {
+                await session.abandon();
+                throw e;
+            }
+            summary.ran++;
+            return r;
+        },
+        receiptStaleness: (args) => store.receiptStaleness(args),
+        [SQL_ACTIVITY]: async (args) => {
+            const sql = args.sql;
+            if (typeof sql !== "string" || sql.trim() === "")
+                throw new Error(`${SQL_ACTIVITY}: needs { sql: "<statement>" }`);
+            const settings = isRecord(args.settings) ? Object.fromEntries(Object.entries(args.settings).map(([k, v]) => [k, String(v)])) : undefined;
+            run.log?.(`    ${sql.split("\n")[0] ?? ""}`);
+            return { rows: await session.sql(sql, settings) };
+        },
+    };
+    try {
+        const records = await runOp(config, {
+            cwd: run.projectDir,
+            environment: run.environment,
+            activities,
+            ...(run.base ? { base: run.base } : {}),
+            ...(run.signal ? { signal: run.signal } : {}),
+            ...(run.onRecord ? { onRecord: run.onRecord } : {}),
+        });
+        return { summary, records };
+    }
+    catch (e) {
+        if (e instanceof StepOpFailed)
+            e.summary = summary;
+        throw e;
+    }
+    finally {
+        // A batch stopped part way (a failed step that was not ours, an abort) leaves nothing behind.
+        await session.abandon();
+    }
+}
+//# sourceMappingURL=backfill.js.map

package/dist/steps/backfill.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"backfill.js","sourceRoot":"","sources":["../../src/steps/backfill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAOzC,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAI3C,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,EAAqB,MAAM,UAAU,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,WAAW,EAAqB,MAAM,YAAY,CAAC;AAE3E,gGAAgG;AAChG,MAAM,CAAC,MAAM,YAAY,GAAG,UAAU,CAAC;AAEvC,2DAA2D;AAC3D,MAAM,UAAU,YAAY,CAAC,aAAqB,EAAE,IAAI,GAAG,aAAa;IACtE,OAAO;QACL,IAAI,EAAE,IAAI;QACV,MAAM,EAAE,UAAU;QAClB,IAAI,EAAE,iBAAiB;QACvB,IAAI,EAAE,aAAa;QACnB,EAAE,EAAE,WAAW;QACf,UAAU,EAAE,KAAK,IAAI,EAAE;QACvB,OAAO,EAAE,EAAE,IAAI,EAAE;QACjB,WAAW,EAAE,sBAAsB,IAAI,gCAAgC;QACvE,OAAO,EAAE,EAAE;QACX,MAAM,EAAE,oDAAoD,IAAI,0HAA0H;KAC3L,CAAC;AACJ,CAAC;AAED,uFAAuF;AACvF,MAAM,CAAC,MAAM,aAAa,GAAG,uBAAuB,CAAC;AAErD,oFAAoF;AACpF,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,UAAsB,YAAY;IAC/E,MAAM,IAAI,GACR,OAAO,KAAK,UAAU;QACpB,CAAC,CAAC,sDAAsD,IAAI,2BAA2B;QACvF,CAAC,CAAC,4CAA4C,IAAI,oBAAoB,CAAC;IAC3E,OAAO;+BACsB,IAAI;;;sDAI/B,OAAO,KAAK,UAAU;QACpB,CAAC,CAAC;;iCAEyB;QAC3B,CAAC,CAAC;8DAEN;;;;;;;;;;;;;2BAayB,IAAI,yBAAyB,IAAI;;gBAE5C,YAAY,eAAe,IAAI;;;;oBAI3B,IAAI;sCACc,IAAI;;;CAGzC,CAAC;AACF,CAAC;AAED,MAAM,QAAQ,GAAG,CAAC,CAAU,EAAgC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,CAAC;AAEnG,uEAAuE;AACvE,MAAM,UAAU,UAAU,CAAC,GAA4B;IACrD,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QACxD,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;YAAE,OAAO,KAA4B,CAAC;IAC5H,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,qHAAqH;AACrH,MAAM,UAAU,gBAAgB,CAAC,MAAgB;IAC/C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC7D,KAAK,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;QACjC,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ;YAAE,QAAQ,CAAC,IAAI,CAAC,UAAU,KAAK,WAAW,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,oGAAoG,CAAC,CAAC;IAC7O,CAAC;IACD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,kBAAkB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,mDAAmD,CAAC,CAAC;IAC3H,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,yIAAyI;AACzI,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,IAAa;IAC5D,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,0DAA0D,CAAC,CAAC;IAC/G,MAAM,IAAI,GAAG,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC7E,IAAI,GAA4B,CAAC;IACjC,IAAI,CAAC;QACH,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,UAAU,OAAO,EAAE,CAAC,CAA4B,CAAC;IAClG,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,wBAAwB,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpG,CAAC;IACD,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,qEAAqE,CAAC,CAAC;IAChH,MAAM,QAAQ,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjF,OAAO,MAAM,CAAC;AAChB,CAAC;AAeD,4GAA4G;AAC5G,MAAM,UAAU,aAAa,CAAC,KAAmB,EAAE,MAAc;IAC/D,OAAO;QACL,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;QACjE,KAAK,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,EAAE,WAAW,CAAC;KAC9F,CAAC;AACJ,CAAC;AAKD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,MAAqB,EAAE,WAAmB;IAC7E,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,uCAAuC,CAAC,CAAC;IAClF,MAAM,KAAK,GACT,YAAY,IAAI,MAAM;QACpB,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,UAAU,EAAE,EAAE;QACjD,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAI,GAAW,EAAE,MAA2B,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAI,GAAG,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;IAC3K,MAAM,KAAK,GAAG,eAAe,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;IACrF,IAAI,UAAU,IAAI,MAAM;QAAE,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;IAC/C,OAAO,KAAK,CAAC;AACf,CAAC;AAgBD,SAAS,iBAAiB,CAAC,IAAoB;IAC7C,OAAO;QACL,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE;QAC7B,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM;QAC1F,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE;QAC7B,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE;KACjC,CAAC;AACJ,CAAC;AAED,mGAAmG;AACnG,SAAS,eAAe,CAAC,MAAwB;IAC/C,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;IACpC,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,oIAAoI;IACpI,IAAI,IAAwB,CAAC;IAC7B,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QAC1B,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,IAAI,GAAG,KAAK,CAAC;QACb,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACxD,CAAC,CAAC;IACF,OAAO;QACL,KAAK,CAAC,IAAI;YACR,MAAM,QAAQ,EAAE,CAAC;YACjB,IAAI,GAAG,SAAS,CAAC;QACnB,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ;YACrB,IAAI,IAAI,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,IAAI,uDAAuD,CAAC,CAAC;YAClJ,IAAI,CAAC;gBACH,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC5B,IAAI,GAAG,IAAI,CAAC;oBACZ,MAAM,MAAM,CAAC,KAAK,CAAC,6BAA6B,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;oBAC9G,MAAM,MAAM,CAAC,KAAK,CAAC,kCAAkC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBACrH,CAAC;gBACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;oBAAE,MAAM,MAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC5I,OAAO,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;YAC1C,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAClD,MAAM,QAAQ,EAAE,CAAC;gBACjB,MAAM,CAAC,CAAC;YACV,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI;YACR,IAAI,CAAC,IAAI;gBAAE,OAAO;YAClB,IAAI,GAAG,KAAK,CAAC;YACb,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,EAAE,QAAQ;KAClB,CAAC;AACJ,CAAC;AAiBD,gFAAgF;AAChF,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,GAAuB;IACrE,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IACjE,MAAM,OAAO,GAAoB,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;IACjK,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAClH,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,UAAU,CAAC,gFAAgF,CAAC,CAAC;IACrH,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,mCAAmC,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,iBAAiB,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;IACpF,MAAM,UAAU,GAA+B;QAC7C,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAC1B,uDAAuD;YACvD,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;YACrB,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,IAA0D,CAAC,CAAC;YAC9F,IAAI,CAAC,CAAC,OAAO;gBAAE,OAAO,CAAC,OAAO,EAAE,CAAC;YACjC,OAAO,CAAC,CAAC;QACX,CAAC;QACD,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAC3B,IAAI,CAAiD,CAAC;YACtD,IAAI,CAAC;gBACH,CAAC,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,IAA2D,CAAC,CAAC;gBAC1F,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;YACvB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;gBACxB,MAAM,CAAC,CAAC;YACV,CAAC;YACD,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,CAAC;QACX,CAAC;QACD,gBAAgB,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAA+D,CAAC;QACnH,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;YACrB,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,GAAG,YAAY,gCAAgC,CAAC,CAAC;YACnH,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAA4B,CAAC,CAAC,CAAC,SAAS,CAAC;YACrK,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC7C,OAAO,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;QACpD,CAAC;KACF,CAAC;IACF,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE;YAClC,GAAG,EAAE,GAAG,CAAC,UAAU;YACnB,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,UAAU;YACV,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7C,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpD,CAAC,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC9B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,YAAY;YAAE,CAAC,CAAC,OAAO,GAAG,OAAO,CAAC;QACnD,MAAM,CAAC,CAAC;IACV,CAAC;YAAS,CAAC;QACT,8FAA8F;QAC9F,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC"}

package/dist/steps/index.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Op steps inside a migration (#12): what `yodel apply` runs besides SQL.
+ *
+ *   planStep(step)            statement, rebuild, backfill, or refused (with why)
+ *   stepSha(step, files)      the hash a step's history rows record
+ *   runRebuild(...)           a ClickHouseRebuildOp step (./rebuild.ts)
+ *   runBackfill(...)          a backfill step, an Op of effect() batches (./backfill.ts)
+ *   runPgMigration(...)       a PostgresMigrationOp step (./postgres-migration.ts)
+ *   defaultStepRunner         what the versioned apply calls for an Op step
+ */
+import type { ReceiptStore } from "@intentius/chant/op/receipt-store";
+import type { ClickHouseEndpoint } from "@intentius/chant-lexicon-sql/clickhouse/http";
+import type { ClickHousePort } from "../history/index.js";
+import type { Migration } from "../migrations/index.js";
+import { type PgBackfillServer } from "./backfill.js";
+import type { StepPlan } from "./kinds.js";
+export * from "./kinds.js";
+export * from "./run.js";
+export * from "./rebuild.js";
+export * from "./backfill.js";
+export * from "./postgres-migration.js";
+/** What running one Op step needs. */
+export interface StepContext {
+    environment: string;
+    /** The project directory, which holds chant.config.ts. */
+    projectDir: string;
+    /** The ClickHouse server a backfill's SQL runs on; absent on Postgres. */
+    port?: ClickHousePort | undefined;
+    /** Postgres: the apply's connection a backfill's SQL runs on, its timeouts, and the schema its receipts are kept in (the history's). */
+    postgres?: (PgBackfillServer & {
+        receiptsSchema: string;
+    }) | undefined;
+    /** Postgres: the environment's lock timeout, for a PostgresMigrationOp step whose options set none. */
+    lockTimeoutMs?: number | undefined;
+    /** The server, for the backfill's receipt store; the port is enough for everything else. */
+    endpoint?: ClickHouseEndpoint | undefined;
+    migration: Migration;
+    index: number;
+    plan: Exclude<StepPlan, {
+        run: "statement";
+    } | {
+        run: "refused";
+    }>;
+    log: (line: string) => void;
+    /** A backfill's receipt store; default the sql lexicon's on the server. */
+    receipts?: ReceiptStore | undefined;
+}
+/** Runs one Op step to the end and returns the summary its history row records. Throws `StepOpFailed` when it stops part way. */
+export type StepRunner = (ctx: StepContext) => Promise<{
+    summary: unknown;
+}>;
+export declare const defaultStepRunner: StepRunner;

package/dist/steps/index.js

@@ -0,0 +1,52 @@
+/**
+ * Op steps inside a migration (#12): what `yodel apply` runs besides SQL.
+ *
+ *   planStep(step)            statement, rebuild, backfill, or refused (with why)
+ *   stepSha(step, files)      the hash a step's history rows record
+ *   runRebuild(...)           a ClickHouseRebuildOp step (./rebuild.ts)
+ *   runBackfill(...)          a backfill step, an Op of effect() batches (./backfill.ts)
+ *   runPgMigration(...)       a PostgresMigrationOp step (./postgres-migration.ts)
+ *   defaultStepRunner         what the versioned apply calls for an Op step
+ */
+import { YodelError } from "../project.js";
+import { runBackfill, serverReceipts } from "./backfill.js";
+import { rebuildChantConfig, runRebuild } from "./rebuild.js";
+import { runPgMigration } from "./postgres-migration.js";
+export * from "./kinds.js";
+export * from "./run.js";
+export * from "./rebuild.js";
+export * from "./backfill.js";
+export * from "./postgres-migration.js";
+export const defaultStepRunner = async (ctx) => {
+    if (ctx.plan.run === "rebuild") {
+        const config = await rebuildChantConfig(ctx.projectDir, ctx.environment);
+        const { summary } = await runRebuild(ctx.plan.options, { environment: ctx.environment, projectDir: ctx.projectDir, schema: ctx.migration.document.schema, log: ctx.log, config });
+        return { summary };
+    }
+    if (ctx.plan.run === "pgMigration") {
+        const { summary } = await runPgMigration(ctx.plan.options, { environment: ctx.environment, projectDir: ctx.projectDir, schema: ctx.migration.document.schema, lockTimeoutMs: ctx.lockTimeoutMs, log: ctx.log });
+        return { summary };
+    }
+    if (!ctx.port && !ctx.postgres)
+        throw new YodelError("a backfill step runs its SQL on the environment's server, and this environment has none");
+    const receipts = ctx.receipts ??
+        (ctx.postgres
+            ? await serverReceipts({ postgres: ctx.postgres.client, schema: ctx.postgres.receiptsSchema }, ctx.environment)
+            : ctx.endpoint
+                ? await serverReceipts({ clickhouse: ctx.endpoint }, ctx.environment)
+                : undefined);
+    if (!receipts)
+        throw new YodelError("a backfill step needs a receipt store, and this environment has no server endpoint for one");
+    const { summary } = await runBackfill(ctx.plan.file, {
+        environment: ctx.environment,
+        projectDir: ctx.projectDir,
+        migrationId: ctx.migration.id,
+        dir: ctx.migration.dir,
+        text: ctx.migration.files.extra?.[ctx.plan.file],
+        ...(ctx.postgres ? { postgres: { client: ctx.postgres.client, timeouts: ctx.postgres.timeouts } } : { port: ctx.port }),
+        receipts,
+        log: ctx.log,
+    });
+    return { summary };
+};
+//# sourceMappingURL=index.js.map

package/dist/steps/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/steps/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,cAAc,EAAyB,MAAM,eAAe,CAAC;AAEnF,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAEzD,cAAc,YAAY,CAAC;AAC3B,cAAc,UAAU,CAAC;AACzB,cAAc,cAAc,CAAC;AAC7B,cAAc,eAAe,CAAC;AAC9B,cAAc,yBAAyB,CAAC;AA0BxC,MAAM,CAAC,MAAM,iBAAiB,GAAe,KAAK,EAAE,GAAG,EAAE,EAAE;IACzD,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;QACzE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;QAClL,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,aAAa,EAAE,CAAC;QACnC,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QAChN,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ;QAAE,MAAM,IAAI,UAAU,CAAC,yFAAyF,CAAC,CAAC;IAChJ,MAAM,QAAQ,GACZ,GAAG,CAAC,QAAQ;QACZ,CAAC,GAAG,CAAC,QAAQ;YACX,CAAC,CAAC,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,cAAc,EAAE,EAAE,GAAG,CAAC,WAAW,CAAC;YAC/G,CAAC,CAAC,GAAG,CAAC,QAAQ;gBACZ,CAAC,CAAC,MAAM,cAAc,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,GAAG,CAAC,WAAW,CAAC;gBACrE,CAAC,CAAC,SAAS,CAAC,CAAC;IACnB,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,UAAU,CAAC,4FAA4F,CAAC,CAAC;IAClI,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE;QACnD,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,WAAW,EAAE,GAAG,CAAC,SAAS,CAAC,EAAE;QAC7B,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,GAAG;QACtB,IAAI,EAAE,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;QAChD,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;QACvH,QAAQ;QACR,GAAG,EAAE,GAAG,CAAC,GAAG;KACb,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,CAAC;AACrB,CAAC,CAAC"}

package/dist/steps/kinds.d.ts

@@ -0,0 +1,101 @@
+/**
+ * What each step of a migration is to `yodel apply`, and the hash its
+ * history rows record.
+ *
+ * A migration's `steps` (in `migration.json`) are chant's: a `statement`
+ * apply sends as SQL, an `op` step naming the Op that makes a change no
+ * statement makes, or a `manual` step a person acts on. Apply runs two kinds
+ * of Op step itself, inside the migration and under its lock and gate:
+ *
+ * - `ClickHouseRebuildOp`, the step `yodel new` writes for a ClickHouse sort
+ *   key, engine or key-column change (`./rebuild.ts`);
+ * - `backfill`, a data backfill the author writes as an Op of `effect()`
+ *   steps in a module in the migration's directory (`./backfill.ts`);
+ * - `PostgresMigrationOp`, the step `yodel new` writes for a Postgres column
+ *   rename or type change across kinds (`./postgres-migration.ts`, #13).
+ *
+ * Everything else (a manual step, a rebuild in app mode) is refused before
+ * anything runs, with exit 2.
+ */
+import type { MigrationStep } from "@intentius/chant-lexicon-sql/migration-statements";
+import { type MigrationFiles } from "../migrations/document.js";
+/** The `op` of a backfill step. */
+export declare const BACKFILL_OP = "backfill";
+/** The module a backfill step runs, in the migration's directory, unless its `options.file` names another. */
+export declare const BACKFILL_FILE = "backfill.ts";
+export declare const REBUILD_OP = "ClickHouseRebuildOp";
+export declare const PG_MIGRATION_OP = "PostgresMigrationOp";
+type OpStep = Extract<MigrationStep, {
+    kind: "op";
+}>;
+/**
+ * A backfill step in `migration.json`. chant's `OpStep` shape, so every
+ * reader of steps sees an Op step:
+ *
+ *     { "kind": "op", "object": "backfill", "type": "Yodel::Backfill", "name": "<migration name>",
+ *       "op": "backfill", "importPath": "./backfill.ts", "options": { "file": "backfill.ts" },
+ *       "declaration": "...", "changes": [], "detail": "..." }
+ */
+export interface BackfillStep extends Omit<OpStep, "op" | "options"> {
+    op: typeof BACKFILL_OP;
+    options: {
+        file: string;
+    };
+}
+export type DualWrite = {
+    mode: "materialized-view";
+    cutoverColumn: string;
+    cutoverDelay?: string;
+} | {
+    mode: "app";
+};
+export interface RebuildOptions {
+    name: string;
+    table: string;
+    dualWrite: DualWrite;
+}
+/** A `PostgresMigrationOp` step's options: chant's `{ name, env, table, column }`, and what the author may add (`using`, `retain`, ...). */
+export interface PgMigrationOptions {
+    name: string;
+    table: string;
+    column: string;
+    using?: string;
+    /** How long the old column is kept after the switch; default none, so the contract runs in the same apply. */
+    retain?: string;
+    batchSize?: number;
+    replicationLag?: {
+        max?: string;
+        wait?: string;
+    } | false;
+    lockTimeoutMs?: number;
+    statementTimeoutMs?: number;
+}
+export type StepPlan = {
+    run: "statement";
+} | {
+    run: "rebuild";
+    options: RebuildOptions;
+} | {
+    run: "pgMigration";
+    options: PgMigrationOptions;
+} | {
+    run: "backfill";
+    file: string;
+} | {
+    run: "refused";
+    reason: string;
+};
+/** The step as a backfill step, when it is one. */
+export declare const asBackfill: (step: MigrationStep | BackfillStep) => BackfillStep | undefined;
+/** How `yodel apply` treats one step. */
+export declare function planStep(step: MigrationStep | BackfillStep): StepPlan;
+/** One line naming a step, for logs and refusals. */
+export declare function describeStep(step: MigrationStep | BackfillStep): string;
+/**
+ * The hash a step's history rows record: for a statement, its SQL's (the
+ * history's `statementSha`); for an Op step, its canonical JSON and, for a
+ * backfill, the text of the module it runs. A step that ran and has since
+ * changed is how a part-way migration's edit is caught.
+ */
+export declare function stepSha(step: MigrationStep | BackfillStep, files?: Pick<MigrationFiles, "extra">): string;
+export {};