@intentius/sql-yodeler 0.0.0-stage → 0.1.0

package/dist/versioned/digest.d.ts

@@ -0,0 +1,77 @@
+/**
+ * The plan digest a versioned apply is approved under.
+ *
+ * It covers what the run would do and the state it would start from:
+ *
+ * - the migrations it would apply, in order, each with its files' checksum
+ *   and the statements of it that already succeeded (a part-way one resumes);
+ * - the history it starts from: every applied migration and its checksum;
+ * - the topology the statements are rendered for (#9): an approval given
+ *   for a single node never applies on a cluster;
+ * - the live schema: a fingerprint of the server's catalog in the
+ *   environment's databases (every table, view and dictionary there, by its
+ *   `create_table_query`, and each database's engine). The history database
+ *   is left out, so recording a run does not move the digest.
+ *
+ * It is chant's plan digest (`computePlanDigest`, kind `yodel-migrations`),
+ * so `chant approve <op> <gate> --plan <digest>` takes it. A change to any of
+ * it after approval (a new pending migration, an edited one, a table changed
+ * out of band, another run applying first) is a new digest, and the
+ * approval does not carry over.
+ */
+import type { ClickHousePort } from "../history/index.js";
+import type { AppliedMigration, PendingMigration } from "./state.js";
+import type { Topology } from "../topology/render.js";
+export declare const DIGEST_KIND = "yodel-migrations";
+export interface LiveScope {
+    /** `sql.profiles.<env>.databases`; every database but the server's own when undefined. */
+    databases: string[] | undefined;
+    /** Left out of the fingerprint. */
+    historyDatabase: string;
+}
+export interface LiveCatalog {
+    databases: Array<{
+        name: string;
+        engine: string;
+    }>;
+    tables: Array<{
+        database: string;
+        name: string;
+        engine: string;
+        create: string;
+    }>;
+}
+/** Reads the catalog the fingerprint covers. */
+export declare function readLiveCatalog(port: ClickHousePort, scope: LiveScope): Promise<LiveCatalog>;
+/** `sha256:<hex>` over the catalog's canonical JSON. */
+export declare function catalogFingerprint(catalog: LiveCatalog): string;
+/** What the digest is taken over. */
+export interface DigestSubject {
+    environment: string;
+    historyDatabase: string;
+    /** The environment's topology, with a cluster's Keeper path and replica name when it sets them. Always set by {@link digestSubject}; absent in a subject carried from before #9. */
+    topology?: Topology;
+    applied: Array<{
+        id: string;
+        checksum: string;
+    }>;
+    pending: Array<{
+        id: string;
+        checksum: string;
+        done: number[];
+    }>;
+    live: string;
+}
+export declare function digestSubject(input: {
+    environment: string;
+    historyDatabase: string;
+    /** Default: a single node. */
+    topology?: Topology | undefined;
+    applied: readonly AppliedMigration[];
+    toApply: readonly PendingMigration[];
+    live: string;
+}): DigestSubject;
+/** The plan digest of a subject, as chant computes and compares them. */
+export declare function planDigest(subject: DigestSubject): Promise<string>;
+/** Whether two digests name the same plan, by chant's rule. */
+export declare function sameDigest(a: string | undefined, b: string | undefined): Promise<boolean>;

package/dist/versioned/digest.js

@@ -0,0 +1,69 @@
+/**
+ * The plan digest a versioned apply is approved under.
+ *
+ * It covers what the run would do and the state it would start from:
+ *
+ * - the migrations it would apply, in order, each with its files' checksum
+ *   and the statements of it that already succeeded (a part-way one resumes);
+ * - the history it starts from: every applied migration and its checksum;
+ * - the topology the statements are rendered for (#9): an approval given
+ *   for a single node never applies on a cluster;
+ * - the live schema: a fingerprint of the server's catalog in the
+ *   environment's databases (every table, view and dictionary there, by its
+ *   `create_table_query`, and each database's engine). The history database
+ *   is left out, so recording a run does not move the digest.
+ *
+ * It is chant's plan digest (`computePlanDigest`, kind `yodel-migrations`),
+ * so `chant approve <op> <gate> --plan <digest>` takes it. A change to any of
+ * it after approval (a new pending migration, an edited one, a table changed
+ * out of band, another run applying first) is a new digest, and the
+ * approval does not carry over.
+ */
+import { createHash } from "node:crypto";
+import { literal } from "../history/index.js";
+export const DIGEST_KIND = "yodel-migrations";
+/** The databases the server keeps for itself, never part of an environment's schema. */
+const SYSTEM_DATABASES = ["system", "information_schema", "INFORMATION_SCHEMA"];
+/** Reads the catalog the fingerprint covers. */
+export async function readLiveCatalog(port, scope) {
+    const where = scope.databases
+        ? `name IN (${scope.databases.filter((d) => d !== scope.historyDatabase).map(literal).join(", ") || "''"})`
+        : `name NOT IN (${[...SYSTEM_DATABASES, scope.historyDatabase].map(literal).join(", ")})`;
+    const databases = await port.query(`SELECT name, engine FROM system.databases WHERE ${where} ORDER BY name`);
+    const names = databases.map((d) => d.name);
+    const tables = names.length === 0
+        ? []
+        : await port.query(`SELECT database, name, engine, create_table_query AS create FROM system.tables WHERE database IN (${names.map(literal).join(", ")}) AND NOT is_temporary ORDER BY database, name`);
+    return { databases, tables };
+}
+/** `sha256:<hex>` over the catalog's canonical JSON. */
+export function catalogFingerprint(catalog) {
+    const canonical = {
+        databases: [...catalog.databases].sort((a, b) => a.name.localeCompare(b.name)).map((d) => [d.name, d.engine]),
+        tables: [...catalog.tables]
+            .sort((a, b) => a.database.localeCompare(b.database) || a.name.localeCompare(b.name))
+            .map((t) => [t.database, t.name, t.engine, t.create]),
+    };
+    return `sha256:${createHash("sha256").update(JSON.stringify(canonical), "utf8").digest("hex")}`;
+}
+export function digestSubject(input) {
+    return {
+        environment: input.environment,
+        historyDatabase: input.historyDatabase,
+        topology: input.topology ?? { kind: "single" },
+        applied: input.applied.map((a) => ({ id: a.id, checksum: a.checksum })),
+        pending: input.toApply.map((p) => ({ id: p.id, checksum: p.checksum, done: p.partWay?.done ?? [] })),
+        live: input.live,
+    };
+}
+/** The plan digest of a subject, as chant computes and compares them. */
+export async function planDigest(subject) {
+    const { computePlanDigest } = await import("@intentius/chant/lifecycle/plan-digest");
+    return computePlanDigest(DIGEST_KIND, subject);
+}
+/** Whether two digests name the same plan, by chant's rule. */
+export async function sameDigest(a, b) {
+    const { samePlanDigest } = await import("@intentius/chant/lifecycle/plan-digest");
+    return samePlanDigest(a, b);
+}
+//# sourceMappingURL=digest.js.map

package/dist/versioned/digest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"digest.js","sourceRoot":"","sources":["../../src/versioned/digest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAI9C,MAAM,CAAC,MAAM,WAAW,GAAG,kBAAkB,CAAC;AAE9C,wFAAwF;AACxF,MAAM,gBAAgB,GAAG,CAAC,QAAQ,EAAE,oBAAoB,EAAE,oBAAoB,CAAC,CAAC;AAchF,gDAAgD;AAChD,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAoB,EAAE,KAAgB;IAC1E,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS;QAC3B,CAAC,CAAC,YAAY,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG;QAC3G,CAAC,CAAC,gBAAgB,CAAC,GAAG,gBAAgB,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;IAC5F,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAmC,mDAAmD,KAAK,gBAAgB,CAAC,CAAC;IAC/I,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,MAAM,GACV,KAAK,CAAC,MAAM,KAAK,CAAC;QAChB,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CACd,qGAAqG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,gDAAgD,CACnL,CAAC;IACR,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AAC/B,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,kBAAkB,CAAC,OAAoB;IACrD,MAAM,SAAS,GAAG;QAChB,SAAS,EAAE,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;QAC7G,MAAM,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;aACxB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;aACpF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;KACxD,CAAC;IACF,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;AAClG,CAAC;AAaD,MAAM,UAAU,aAAa,CAAC,KAQ7B;IACC,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC9C,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QACvE,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;QACpG,IAAI,EAAE,KAAK,CAAC,IAAI;KACjB,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAAsB;IACrD,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,wCAAwC,CAAC,CAAC;IACrF,OAAO,iBAAiB,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;AACjD,CAAC;AAED,+DAA+D;AAC/D,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,CAAqB,EAAE,CAAqB;IAC3E,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,wCAAwC,CAAC,CAAC;IAClF,OAAO,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC9B,CAAC"}

package/dist/versioned/driver.d.ts

@@ -0,0 +1,41 @@
+/**
+ * What the versioned apply needs from a dialect: its lock, the live-schema
+ * fingerprint the plan digest covers, and how one statement is sent and
+ * recorded. The engine (`./engine.ts`) is the same for both dialects: it
+ * reads the history, compares it with the directory, checks the digest and
+ * the approval, and walks the statements and Op steps in order, recording
+ * each one's start and failure; the driver does the dialect's part.
+ *
+ *   ClickHouse  ./clickhouse.ts   the Keeper (or local) lock, system.tables, a statement then its row
+ *   Postgres    ./postgres/       the session advisory lock, chant's catalog reader, a statement and its row in one transaction
+ */
+import type { SqlDialect } from "../project.js";
+import type { StatementStep } from "@intentius/chant-lexicon-sql/migration-statements";
+import type { Lock } from "../history/index.js";
+import type { StepContext } from "../steps/index.js";
+export interface StatementRun {
+    /**
+     * Writes the statement's `succeeded` row, with `note` (the retries it
+     * took, an INVALID index it dropped; `""` when there is nothing to say).
+     * The driver calls it where the row commits with the statement when the
+     * dialect can: inside the statement's transaction on Postgres.
+     */
+    succeeded: (note: string) => Promise<unknown>;
+    log: (line: string) => void;
+}
+/** One dialect's part of the versioned apply. A failed statement throws with the server's message, which the history's failed row records. */
+export interface VersionedDriver {
+    dialect: SqlDialect;
+    /** The `note` a statement's rows carry: what ClickHouse rendered it for (#9); none on Postgres. */
+    rowNote?: string;
+    /** Takes the environment's apply lock, or throws a `LockHeldError` (exit 5). */
+    acquireLock(holder: string): Promise<Lock>;
+    /** A fingerprint of the live schema in the environment's scope, the history left out. */
+    liveFingerprint(): Promise<string>;
+    /** Sends one statement and writes its `succeeded` row through `run`; throws with the server's message when it fails. */
+    runStatement(step: StatementStep, run: StatementRun): Promise<void>;
+    /** What an Op step's runner gets from the dialect: the ClickHouse port and endpoint; the Postgres lock timeout, and the connection, timeouts and receipts schema a backfill uses. */
+    stepContext(): Pick<StepContext, "port" | "endpoint" | "lockTimeoutMs" | "postgres">;
+    /** Closes what the driver opened (a Postgres connection). Safe to call twice. */
+    close(): Promise<void>;
+}

package/dist/versioned/driver.js

@@ -0,0 +1,13 @@
+/**
+ * What the versioned apply needs from a dialect: its lock, the live-schema
+ * fingerprint the plan digest covers, and how one statement is sent and
+ * recorded. The engine (`./engine.ts`) is the same for both dialects: it
+ * reads the history, compares it with the directory, checks the digest and
+ * the approval, and walks the statements and Op steps in order, recording
+ * each one's start and failure; the driver does the dialect's part.
+ *
+ *   ClickHouse  ./clickhouse.ts   the Keeper (or local) lock, system.tables, a statement then its row
+ *   Postgres    ./postgres/       the session advisory lock, chant's catalog reader, a statement and its row in one transaction
+ */
+export {};
+//# sourceMappingURL=driver.js.map

package/dist/versioned/driver.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"driver.js","sourceRoot":"","sources":["../../src/versioned/driver.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG"}

package/dist/versioned/engine.d.ts

@@ -0,0 +1,152 @@
+/**
+ * The versioned apply: pending migrations, statement by statement, each
+ * statement's start and outcome appended to the history, under the lock,
+ * after the plan digest the run was approved under is checked again.
+ *
+ * Statements are the `statement` steps of each migration's `migration.json`,
+ * written for a single node, each rendered for the environment's topology
+ * as it is sent (#9, `../topology/render.ts`). A statement whose latest
+ * history row is `succeeded` is never sent again, so a migration that failed
+ * part way resumes at the statement that failed. A statement whose latest
+ * row is `started` (the runner stopped while it ran) is sent again: the
+ * history cannot tell whether the server ran it.
+ *
+ * An Op step (#12) runs in its place among the statements: the statements
+ * before it first, the ones after it only once it succeeded. A
+ * `ClickHouseRebuildOp` step runs chant's rebuild Op, a backfill step the
+ * author's Op of `effect()` batches (`../steps/`), each in this process
+ * under the same lock, and behind the same approval: the step and any file
+ * it runs are in the migration's checksum, so in the plan digest. Its rows
+ * are `kind = 'step'`, with the step's summary (receipts skipped, work done)
+ * in `note`. A step that failed or was stopped runs again on the next run
+ * and resumes from its receipts: the partitions or batches that finished
+ * are skipped.
+ *
+ * On Postgres (#13) a `PostgresMigrationOp` step (a column rename or type
+ * change) runs the same way, as chant's expand-and-contract Op
+ * (`../steps/postgres-migration.ts`).
+ *
+ * What is the dialect's (the lock, the live-schema fingerprint, how a
+ * statement is sent and its `succeeded` row written) is the driver's
+ * (`./driver.ts`): on Postgres a transactional statement and its row commit
+ * in one transaction, under `lock_timeout`, retried on a lock timeout
+ * (`./postgres/executor.ts`).
+ *
+ * Manual steps, and Op steps apply does not run (a rebuild in app mode, a
+ * step of the other dialect), are refused before anything runs (exit 2).
+ */
+import type { ClickHouseEndpoint } from "@intentius/chant-lexicon-sql/clickhouse/http";
+import { YodelError, type Environment, type Project } from "../project.js";
+import type { Migration } from "../migrations/index.js";
+import type { Topology } from "../topology/render.js";
+import { type StepRunner } from "../steps/index.js";
+import type { ReceiptStore } from "@intentius/chant/op/receipt-store";
+import { type ClickHousePort, type HistoryLocation, type HistoryState, type HistoryStoreLike } from "../history/index.js";
+import type { SqlDialect } from "../project.js";
+import type { VersionedDriver } from "./driver.js";
+import { type PgConnect } from "./postgres/driver.js";
+import { type PendingMigration, type VersionedStatus } from "./state.js";
+import { type LiveScope } from "./digest.js";
+/** Exit code: a checksum mismatch, an applied migration gone from the directory, or an out-of-order migration without `--allow-out-of-order`. */
+export declare const REFUSED = 4;
+/** Everything a versioned verb needs about one environment. */
+export interface VersionedEnv {
+    environment: string;
+    dialect: SqlDialect;
+    /** The dialect's lock, fingerprint and statement sending (`./driver.ts`). */
+    driver: VersionedDriver;
+    /** The ClickHouse server; absent on Postgres, whose connections the driver keeps. */
+    port?: ClickHousePort;
+    store: HistoryStoreLike;
+    /** Where the history is: on Postgres, `database` is the history's schema and the topology a single node. */
+    location: HistoryLocation;
+    /** ClickHouse: the databases the fingerprint covers. Postgres: `databases` is the profile's schemas. */
+    scope: LiveScope;
+    /** Identifies the server, for the local lock's file name. */
+    serverKey: string;
+    lockTtlMs: number;
+    /** The project directory: an Op step reads chant.config.ts from it. */
+    projectDir?: string;
+    /** The server, for the receipt store a backfill step keeps its receipts in. */
+    endpoint?: ClickHouseEndpoint;
+    /** What each statement is rendered for when it is sent (`../config/settings.ts`). Default: a single node. */
+    topology?: Topology;
+    /** Where {@link topology} came from: `yodel.config.ts`, `sql.profiles.<env>.topology`, `YODEL_TOPOLOGY` or `default`. */
+    topologySource?: string;
+}
+/**
+ * Binds an environment's server (`sql.profiles.<env>`) and history. `port`
+ * (ClickHouse) and `pgConnect` (Postgres) stand in for the server in tests.
+ * Close it with {@link closeVersioned} when done: on Postgres it holds a
+ * connection.
+ */
+export declare function openVersioned(project: Project, env: Environment, processEnv?: NodeJS.ProcessEnv, port?: ClickHousePort, pgConnect?: PgConnect): Promise<VersionedEnv>;
+/** Closes what {@link openVersioned} opened (a Postgres connection). Safe to call twice. */
+export declare const closeVersioned: (venv: VersionedEnv) => Promise<void>;
+/** The history and how the directory compares with it. Reads only; takes no lock. */
+export declare function inspect(venv: VersionedEnv, ordered: readonly Migration[], options?: {
+    sync?: boolean;
+}): Promise<{
+    state: HistoryState;
+    status: VersionedStatus;
+}>;
+/** The refusal for an integrity problem, or `undefined`. */
+export declare function integrityRefusal(status: VersionedStatus, allowOutOfOrder: boolean): YodelError | undefined;
+/** The refusal for the steps among `toApply` that apply does not run (manual steps, Ops it does not run), or `undefined`. */
+export declare function stepRefusal(toApply: readonly PendingMigration[]): YodelError | undefined;
+/** What a run would apply, in chain order, or the refusal. */
+export declare function selectToApply(status: VersionedStatus, allowOutOfOrder: boolean): PendingMigration[];
+/** The plan digest of applying `toApply` from the server's state now. */
+export declare function currentDigest(venv: VersionedEnv, status: VersionedStatus, toApply: readonly PendingMigration[]): Promise<string>;
+export declare class ApplyFailed extends YodelError {
+    readonly migration: string;
+    readonly statement: number;
+    readonly serverError: string;
+    constructor(migration: string, statement: number, serverError: string);
+}
+/** An Op step failed or stopped part way; the next run resumes it from its receipts. */
+export declare class StepFailed extends YodelError {
+    readonly migration: string;
+    readonly step: number;
+    readonly detail: string;
+    constructor(migration: string, step: number, detail: string);
+}
+export interface ExecuteOptions {
+    /** The digest the Op's gate handed the Apply step. A run whose digest has moved since is refused. */
+    approvedDigest: string;
+    /**
+     * Checks the approval itself, given the digest computed under the lock;
+     * throws to refuse. `--execute` looks it up in chant's gate ledger
+     * (`./approval.ts`). Required: the digest alone is never an approval.
+     */
+    approve: (digest: string) => Promise<unknown>;
+    allowOutOfOrder: boolean;
+    /** Progress, one line at a time. */
+    log: (line: string) => void;
+    /** The silences in force for a migration, as JSON text for the history's `silences` field. Default `"[]"`. */
+    silences?: (m: Migration) => string;
+    processEnv?: NodeJS.ProcessEnv;
+    /** Runs an Op step. Default: chant's local executor (`../steps/`). */
+    runStep?: StepRunner;
+    /** The receipt store a backfill step uses. Default: chant's, on the server. */
+    receipts?: ReceiptStore;
+}
+export interface ExecuteResult {
+    applied: string[];
+    /** Statements not sent because an earlier run had already succeeded with them. */
+    skipped: Array<{
+        id: string;
+        statement: number;
+    }>;
+    lock: {
+        kind: "keeper" | "local" | "advisory";
+        where: string;
+        notice?: string;
+    };
+}
+/**
+ * Applies the pending migrations: takes the lock, reads the history again
+ * under it, refuses on any integrity problem or a digest that moved since
+ * approval, then runs each statement not yet succeeded.
+ */
+export declare function executeApply(venv: VersionedEnv, ordered: readonly Migration[], options: ExecuteOptions): Promise<ExecuteResult>;

package/dist/versioned/engine.js

@@ -0,0 +1,277 @@
+/**
+ * The versioned apply: pending migrations, statement by statement, each
+ * statement's start and outcome appended to the history, under the lock,
+ * after the plan digest the run was approved under is checked again.
+ *
+ * Statements are the `statement` steps of each migration's `migration.json`,
+ * written for a single node, each rendered for the environment's topology
+ * as it is sent (#9, `../topology/render.ts`). A statement whose latest
+ * history row is `succeeded` is never sent again, so a migration that failed
+ * part way resumes at the statement that failed. A statement whose latest
+ * row is `started` (the runner stopped while it ran) is sent again: the
+ * history cannot tell whether the server ran it.
+ *
+ * An Op step (#12) runs in its place among the statements: the statements
+ * before it first, the ones after it only once it succeeded. A
+ * `ClickHouseRebuildOp` step runs chant's rebuild Op, a backfill step the
+ * author's Op of `effect()` batches (`../steps/`), each in this process
+ * under the same lock, and behind the same approval: the step and any file
+ * it runs are in the migration's checksum, so in the plan digest. Its rows
+ * are `kind = 'step'`, with the step's summary (receipts skipped, work done)
+ * in `note`. A step that failed or was stopped runs again on the next run
+ * and resumes from its receipts: the partitions or batches that finished
+ * are skipped.
+ *
+ * On Postgres (#13) a `PostgresMigrationOp` step (a column rename or type
+ * change) runs the same way, as chant's expand-and-contract Op
+ * (`../steps/postgres-migration.ts`).
+ *
+ * What is the dialect's (the lock, the live-schema fingerprint, how a
+ * statement is sent and its `succeeded` row written) is the driver's
+ * (`./driver.ts`): on Postgres a transactional statement and its row commit
+ * in one transaction, under `lock_timeout`, retried on a lock timeout
+ * (`./postgres/executor.ts`).
+ *
+ * Manual steps, and Op steps apply does not run (a rebuild in app mode, a
+ * step of the other dialect), are refused before anything runs (exit 2).
+ */
+import { YodelError } from "../project.js";
+import { PLAN_NEEDS_OP } from "../chant/plan.js";
+import { environmentSettings } from "../config/settings.js";
+import { defaultStepRunner, describeStep, planStep, stepSha, StepOpFailed } from "../steps/index.js";
+import { HistoryRecorder, HistoryStore, httpPort, MIGRATION_ROW, newRunContext, SINGLE, statementDone, statementSha, withLock, appliedBy, } from "../history/index.js";
+import { clickhouseDriver } from "./clickhouse.js";
+import { openPostgres } from "./postgres/driver.js";
+import { compareHistory, outOfOrder } from "./state.js";
+import { digestSubject, planDigest, sameDigest } from "./digest.js";
+/** Exit code: a checksum mismatch, an applied migration gone from the directory, or an out-of-order migration without `--allow-out-of-order`. */
+export const REFUSED = 4;
+/**
+ * Binds an environment's server (`sql.profiles.<env>`) and history. `port`
+ * (ClickHouse) and `pgConnect` (Postgres) stand in for the server in tests.
+ * Close it with {@link closeVersioned} when done: on Postgres it holds a
+ * connection.
+ */
+export async function openVersioned(project, env, processEnv = process.env, port, pgConnect) {
+    const settings = environmentSettings(project, env.name, processEnv);
+    if (project.dialect === "postgres") {
+        const pg = await openPostgres(project, env, processEnv, settings.historyDatabase, pgConnect);
+        return {
+            environment: env.name,
+            dialect: "postgres",
+            driver: pg.driver,
+            store: pg.store,
+            location: { database: settings.historyDatabase, topology: SINGLE },
+            scope: { databases: pg.schemas, historyDatabase: settings.historyDatabase },
+            serverKey: pg.endpoint.url,
+            lockTtlMs: settings.lockTtlMs,
+            projectDir: project.dir,
+        };
+    }
+    const { resolveClickHouseTarget, isUnresolvedTarget } = await import("@intentius/chant-lexicon-sql/clickhouse/live/bind");
+    const target = resolveClickHouseTarget({ environment: env.name, config: project.config, env: processEnv });
+    if (isUnresolvedTarget(target))
+        throw new YodelError(target.detail);
+    const endpoint = target.endpoint;
+    const location = { database: settings.historyDatabase, topology: settings.historyTopology };
+    const p = port ?? (await httpPort(endpoint));
+    const scope = { databases: target.databases, historyDatabase: settings.historyDatabase };
+    return {
+        environment: env.name,
+        dialect: "clickhouse",
+        driver: clickhouseDriver({ port: p, location, scope, serverKey: endpoint.url, lockTtlMs: settings.lockTtlMs, endpoint, topology: settings.topology }),
+        port: p,
+        store: new HistoryStore(p, location),
+        location,
+        scope,
+        serverKey: endpoint.url,
+        lockTtlMs: settings.lockTtlMs,
+        projectDir: project.dir,
+        endpoint,
+        topology: settings.topology,
+        topologySource: settings.topologySource,
+    };
+}
+/** Closes what {@link openVersioned} opened (a Postgres connection). Safe to call twice. */
+export const closeVersioned = (venv) => venv.driver.close();
+/** The history and how the directory compares with it. Reads only; takes no lock. */
+export async function inspect(venv, ordered, options = {}) {
+    const state = await venv.store.state(options);
+    return { state, status: compareHistory(ordered, state) };
+}
+/** The refusal for an integrity problem, or `undefined`. */
+export function integrityRefusal(status, allowOutOfOrder) {
+    if (status.mismatches.length > 0) {
+        return new YodelError(`refused: the history does not match the migrations directory; nothing was applied:\n${status.mismatches.map((m) => `  ${m.message}`).join("\n")}`, REFUSED);
+    }
+    const late = outOfOrder(status);
+    if (late.length > 0 && !allowOutOfOrder) {
+        const newest = status.applied.filter((a) => a.index !== undefined).at(-1)?.id ?? "";
+        return new YodelError(`refused: ${late.map((p) => p.id).join(", ")} ${late.length === 1 ? "is" : "are"} out of order: the chain puts ${late.length === 1 ? "it" : "them"} before ${newest}, which is already applied. ` +
+            `Nothing was applied. Pass --allow-out-of-order to apply ${late.length === 1 ? "it" : "them"} now (in chain order); ${late.length === 1 ? "it is" : "they are"} never skipped.`, REFUSED);
+    }
+    return undefined;
+}
+/** The refusal for the steps among `toApply` that apply does not run (manual steps, Ops it does not run), or `undefined`. */
+export function stepRefusal(toApply) {
+    const lines = toApply.flatMap((p) => p.refusedSteps.map(({ index, reason }) => `  ${p.id} step ${String(index)}: ${reason}`));
+    if (lines.length === 0)
+        return undefined;
+    return new YodelError(`refused: a pending migration has a step yodel apply does not run: a person acts on a manual step, and yodel apply runs only ClickHouseRebuildOp (materialized-view mode) and backfill steps on ClickHouse, and PostgresMigrationOp on Postgres. Nothing was applied:\n${lines.join("\n")}`, PLAN_NEEDS_OP);
+}
+/** What a run would apply, in chain order, or the refusal. */
+export function selectToApply(status, allowOutOfOrder) {
+    const refusal = integrityRefusal(status, allowOutOfOrder) ?? stepRefusal(status.pending);
+    if (refusal)
+        throw refusal;
+    return status.pending;
+}
+/** The plan digest of applying `toApply` from the server's state now. */
+export async function currentDigest(venv, status, toApply) {
+    const live = await venv.driver.liveFingerprint();
+    return planDigest(digestSubject({ environment: venv.environment, historyDatabase: venv.location.database, topology: venv.topology, applied: status.applied, toApply, live }));
+}
+export class ApplyFailed extends YodelError {
+    migration;
+    statement;
+    serverError;
+    constructor(migration, statement, serverError) {
+        super(`${migration} failed at statement ${String(statement)}: ${serverError}\n` +
+            `Fix the cause (on the server, or that statement in the migration's files), then run yodel apply again: ` +
+            `it resumes at statement ${String(statement)} and never sends again the statements that succeeded.`, 1);
+        this.migration = migration;
+        this.statement = statement;
+        this.serverError = serverError;
+        this.name = "ApplyFailed";
+    }
+}
+/** An Op step failed or stopped part way; the next run resumes it from its receipts. */
+export class StepFailed extends YodelError {
+    migration;
+    step;
+    detail;
+    constructor(migration, step, detail) {
+        super(`${migration} failed at step ${String(step)}: ${detail}\n` +
+            `Fix the cause, then run yodel apply again: it resumes at step ${String(step)}, whose finished work (the partitions or batches with a receipt) is not done again, ` +
+            `and never sends again the statements that succeeded.`, 1);
+        this.migration = migration;
+        this.step = step;
+        this.detail = detail;
+        this.name = "StepFailed";
+    }
+}
+/**
+ * Applies the pending migrations: takes the lock, reads the history again
+ * under it, refuses on any integrity problem or a digest that moved since
+ * approval, then runs each statement not yet succeeded.
+ */
+export async function executeApply(venv, ordered, options) {
+    const lock = await venv.driver.acquireLock(appliedBy(options.processEnv));
+    if (lock.notice)
+        options.log(`lock: ${lock.notice}`);
+    else
+        options.log(`lock: held in ${lock.where}`);
+    return withLock(lock, venv.lockTtlMs, async (beat) => {
+        await venv.store.ensure();
+        const { state, status } = await inspect(venv, ordered);
+        const toApply = selectToApply(status, options.allowOutOfOrder);
+        const digest = await currentDigest(venv, status, toApply);
+        if (!(await sameDigest(digest, options.approvedDigest))) {
+            const { describeMove, explainAgainst } = await import("./explain.js");
+            const moved = await explainAgainst(venv, ordered, { approvedDigest: options.approvedDigest });
+            throw new YodelError(`refused: the plan moved since it was approved (approved ${options.approvedDigest}, now ${digest}): ${describeMove(moved)}. ` +
+                `Nothing was applied; run yodel apply ${venv.environment} again and approve the new plan.`, 1);
+        }
+        await options.approve(digest);
+        const recorder = new HistoryRecorder(venv.store, newRunContext(options.approvedDigest, options.processEnv), state.maxSeq);
+        const result = { applied: [], skipped: [], lock: { kind: lock.kind, where: lock.where, ...(lock.notice ? { notice: lock.notice } : {}) } };
+        for (const p of toApply) {
+            await applyMigration(venv, p, state.byId.get(p.id), recorder, beat, options, result);
+            result.applied.push(p.id);
+        }
+        return result;
+    });
+}
+async function applyMigration(venv, p, rec, recorder, beat, options, result) {
+    const m = p.migration;
+    const silences = options.silences?.(m) ?? "[]";
+    const base = { migration_id: m.id, parent: m.parent ?? "", checksum: p.checksum, silences };
+    options.log(`${m.id}: ${p.partWay ? `resuming at statement ${String(p.partWay.resumeAt)}` : "applying"}${p.outOfOrder ? " (out of order, allowed)" : ""}`);
+    await recorder.record({ ...base, kind: "migration", statement_index: MIGRATION_ROW, statement_sha: "", status: "started" });
+    for (const [i, step] of m.document.steps.entries()) {
+        if (statementDone(rec, i)) {
+            options.log(`  statement ${String(i)}: succeeded in an earlier run; not sent`);
+            result.skipped.push({ id: m.id, statement: i });
+            continue;
+        }
+        if (step.kind !== "statement") {
+            await beat();
+            await runOpStep(venv, p, i, recorder, base, options);
+            continue;
+        }
+        await beat();
+        // The neutral statement's sha, so a resume matches on every topology; the note says what it was rendered for.
+        const sha = statementSha(step.sql);
+        const rowNote = venv.driver.rowNote ?? "";
+        const row = { ...base, kind: "statement", statement_index: i, statement_sha: sha, note: rowNote };
+        await recorder.record({ ...row, status: "started" });
+        try {
+            // The driver writes the succeeded row: on Postgres in the statement's own transaction, so the two commit together.
+            await venv.driver.runStatement(step, {
+                succeeded: (note) => recorder.record({ ...row, status: "succeeded", note: [rowNote, note].filter(Boolean).join("; ") }),
+                log: options.log,
+            });
+        }
+        catch (e) {
+            const error = e instanceof Error ? e.message : String(e);
+            await recorder.record({ ...row, status: "failed", error });
+            await recorder.record({ ...base, kind: "migration", statement_index: MIGRATION_ROW, statement_sha: "", status: "failed", error: `statement ${String(i)}: ${error}` });
+            options.log(`  statement ${String(i)}: failed: ${error.split("\n")[0] ?? ""}`);
+            throw new ApplyFailed(m.id, i, error.split("\n")[0] ?? error);
+        }
+        options.log(`  statement ${String(i)}: ok (${step.rule} ${step.class}, ${step.name})`);
+    }
+    await recorder.record({ ...base, kind: "migration", statement_index: MIGRATION_ROW, statement_sha: "", status: "succeeded" });
+    options.log(`${m.id}: applied`);
+}
+/** Runs Op step `i` of `p`, recording its start and outcome. */
+async function runOpStep(venv, p, i, recorder, base, options) {
+    const m = p.migration;
+    const step = m.document.steps[i];
+    if (!step)
+        throw new YodelError(`${m.id} has no step ${String(i)}`);
+    const plan = planStep(step);
+    if (plan.run === "statement" || plan.run === "refused" || p.refusedSteps.some((r) => r.index === i)) {
+        throw stepRefusal([p]) ?? new YodelError(`${m.id} step ${String(i)}: ${plan.run === "refused" ? plan.reason : "not a step yodel apply runs"}`, PLAN_NEEDS_OP);
+    }
+    const row = { ...base, kind: "step", statement_index: i, statement_sha: stepSha(step, m.files) };
+    const resumed = p.partWay !== undefined && p.partWay.resumeAt === i;
+    options.log(`  step ${String(i)}: ${describeStep(step)}${resumed ? " (resuming from its receipts)" : ""}`);
+    await recorder.record({ ...row, status: "started" });
+    const run = options.runStep ?? defaultStepRunner;
+    let summary;
+    try {
+        ({ summary } = await run({
+            environment: venv.environment,
+            projectDir: venv.projectDir ?? process.cwd(),
+            ...venv.driver.stepContext(),
+            migration: m,
+            index: i,
+            plan,
+            log: options.log,
+            receipts: options.receipts,
+        }));
+    }
+    catch (e) {
+        const detail = e instanceof Error ? e.message : String(e);
+        const note = e instanceof StepOpFailed && e.summary !== undefined ? JSON.stringify(e.summary) : "";
+        await recorder.record({ ...row, status: "failed", error: detail, note });
+        await recorder.record({ ...base, kind: "migration", statement_index: MIGRATION_ROW, statement_sha: "", status: "failed", error: `step ${String(i)}: ${detail}` });
+        options.log(`  step ${String(i)}: failed: ${detail.split("\n")[0] ?? ""}${note ? ` (${note})` : ""}`);
+        throw new StepFailed(m.id, i, detail.split("\n")[0] ?? detail);
+    }
+    const note = JSON.stringify(summary ?? {});
+    await recorder.record({ ...row, status: "succeeded", note });
+    options.log(`  step ${String(i)}: ok ${note}`);
+}
+//# sourceMappingURL=engine.js.map