@intentius/chant-lexicon-sql 0.0.0-stage → 0.112.1

package/src/op/activities/clickhouse-rebuild.ts

@@ -0,0 +1,136 @@
+/**
+ * The rebuild migration's activities (#3198), one per step of
+ * `ClickHouseRebuildOp` (`../../clickhouse/rebuild/op.ts`). Each takes the
+ * same arguments, reads the build's declaration of the table and binds the
+ * environment's server the way the applier does (`./clickhouse-apply.ts`),
+ * then runs its step (`../../clickhouse/rebuild/`), which re-reads the server.
+ */
+
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import type { ChantConfig } from "@intentius/chant/config";
+import { currentOpRun, parseDuration } from "@intentius/chant/op";
+import { bindClickHouse } from "../../clickhouse/live/bind";
+import { declaredObjects } from "../../clickhouse/apply/apply";
+import { declaredTable, observeRebuild, rebuildPlanDigest } from "../../clickhouse/rebuild/observe";
+import { backfill, type BackfillDeps, type BackfillResult } from "../../clickhouse/rebuild/backfill";
+import { verifyRebuild, type VerifyResult } from "../../clickhouse/rebuild/verify";
+import {
+  compensate,
+  createNewTable,
+  dropOldTable,
+  retainPlan,
+  startDualWrite,
+  swapTables,
+  type CompensateResult,
+  type CreateResult,
+  type DropResult,
+  type DualWriteResult,
+  type RebuildRun,
+  type RetainResult,
+  type SwapResult,
+} from "../../clickhouse/rebuild/steps";
+import type { ClickHouseRebuildArgs } from "../../clickhouse/rebuild/op";
+import { CLASSIFIER_RULES } from "../../clickhouse/plan/rules";
+import { resolveMarker } from "./clickhouse-apply";
+
+export type { ClickHouseRebuildArgs } from "../../clickhouse/rebuild/op";
+
+/** What a test (or an embedding caller) may inject instead of the project's config and the process env. */
+export interface ClickHouseRebuildDeps {
+  config?: Pick<ChantConfig, "sql" | "ownership">;
+  env?: Record<string, string | undefined>;
+  log?: (line: string) => void;
+  backfill?: BackfillDeps;
+}
+
+async function loadConfig(cwd: string): Promise<Pick<ChantConfig, "sql" | "ownership"> | undefined> {
+  try {
+    const { loadChantConfig } = await import("@intentius/chant/config");
+    return (await loadChantConfig(cwd)).config;
+  } catch {
+    return undefined;
+  }
+}
+
+/** Bind the server and read the declaration: what every step starts from. */
+export async function rebuildRun(args: ClickHouseRebuildArgs, signal: AbortSignal | undefined, deps: ClickHouseRebuildDeps): Promise<RebuildRun> {
+  const cwd = args.cwd ?? process.cwd();
+  const config = deps.config ?? (await loadConfig(cwd));
+  const marker = resolveMarker(args, config);
+  const target = await bindClickHouse({ ...(args.environment !== undefined ? { environment: args.environment } : {}), config: config ?? {}, ...(deps.env ? { env: deps.env } : {}) });
+  const declared = declaredTable(declaredObjects(readFileSync(resolve(cwd, args.buildPath), "utf8"), target.defaultDatabase), args.table);
+  const runId = currentOpRun()?.runId;
+  return {
+    target,
+    declared,
+    ...(marker ? { marker } : {}),
+    dualWrite: args.dualWrite,
+    retainMs: parseDuration(args.retain ?? "7d"),
+    ...(args.mutationTimeout ? { mutationTimeoutMs: parseDuration(args.mutationTimeout) } : {}),
+    ...(args.replicaTimeout ? { replicaTimeoutMs: parseDuration(args.replicaTimeout) } : {}),
+    log: deps.log ?? ((line: string) => console.log(line)),
+    ...(signal ? { signal } : {}),
+    ...(runId ? { runId } : {}),
+  };
+}
+
+export interface RebuildPlanResult {
+  state: "rebuild" | "swapped" | "done";
+  table: string;
+  /** The plan's digest, before the verification; present while there is a rebuild to run. */
+  planDigest?: string;
+  /** Each classified change, as `<rule> <field>: <before> -> <after>`. */
+  changes: string[];
+  summary: string;
+}
+
+/** Plan: classify the table's change against the server and say how far the rebuild has got. */
+export async function clickhouseRebuildPlan(args: ClickHouseRebuildArgs, signal?: AbortSignal, deps: ClickHouseRebuildDeps = {}): Promise<RebuildPlanResult> {
+  const run = await rebuildRun(args, signal, deps);
+  const o = await observeRebuild(run.target, run.declared, run.marker);
+  const changes = o.changes.map((c) => `${c.rule} ${c.field}${c.before !== undefined || c.after !== undefined ? `: ${c.before ?? "none"} -> ${c.after ?? "none"}` : ""}`);
+  const rebuilds = o.changes.filter((c) => c.class === "rebuild").map((c) => `${c.rule} ${CLASSIFIER_RULES[c.rule].title}`);
+  const summary =
+    o.state === "rebuild"
+      ? `${o.names.key} needs a rebuild (${rebuilds.join("; ")}); ${o.copied.length} column(s) copied${o.newTable ? ", new table already made" : ""}`
+      : o.state === "swapped"
+        ? `${o.names.key} has been swapped; the old table is kept as ${o.names.database}.${o.names.oldName}`
+        : `${o.names.key} holds its declaration; nothing to rebuild`;
+  run.log(`-- ${summary}`);
+  for (const c of changes) run.log(`--   ${c}`);
+  return { state: o.state, table: o.names.key, ...(o.state === "rebuild" ? { planDigest: rebuildPlanDigest(o, run.dualWrite) } : {}), changes, summary };
+}
+
+export async function clickhouseRebuildCreate(args: ClickHouseRebuildArgs, signal?: AbortSignal, deps: ClickHouseRebuildDeps = {}): Promise<CreateResult> {
+  return createNewTable(await rebuildRun(args, signal, deps));
+}
+
+export async function clickhouseRebuildDualWrite(args: ClickHouseRebuildArgs, signal?: AbortSignal, deps: ClickHouseRebuildDeps = {}): Promise<DualWriteResult> {
+  return startDualWrite(await rebuildRun(args, signal, deps));
+}
+
+export async function clickhouseRebuildBackfill(args: ClickHouseRebuildArgs, signal?: AbortSignal, deps: ClickHouseRebuildDeps = {}): Promise<BackfillResult> {
+  return backfill(await rebuildRun(args, signal, deps), deps.backfill);
+}
+
+export async function clickhouseRebuildVerify(args: ClickHouseRebuildArgs, signal?: AbortSignal, deps: ClickHouseRebuildDeps = {}): Promise<VerifyResult> {
+  return verifyRebuild(await rebuildRun(args, signal, deps));
+}
+
+export async function clickhouseRebuildSwap(args: ClickHouseRebuildArgs, signal?: AbortSignal, deps: ClickHouseRebuildDeps = {}): Promise<SwapResult> {
+  return swapTables(await rebuildRun(args, signal, deps));
+}
+
+export async function clickhouseRebuildRetain(args: ClickHouseRebuildArgs, signal?: AbortSignal, deps: ClickHouseRebuildDeps = {}): Promise<RetainResult> {
+  return retainPlan(await rebuildRun(args, signal, deps));
+}
+
+export async function clickhouseRebuildDrop(args: ClickHouseRebuildArgs, signal?: AbortSignal, deps: ClickHouseRebuildDeps = {}): Promise<DropResult> {
+  return dropOldTable(await rebuildRun(args, signal, deps));
+}
+
+/** onFailure: drop the new table and the dual-write view. */
+export async function clickhouseRebuildCompensate(args: ClickHouseRebuildArgs, signal?: AbortSignal, deps: ClickHouseRebuildDeps = {}): Promise<CompensateResult> {
+  return compensate(await rebuildRun(args, signal, deps));
+}

package/src/op/activities/index.ts

@@ -0,0 +1,50 @@
+/**
+ * sql Op activities, resolved by the core activity registry when a project's
+ * `chant.config.ts` lists the `sql` lexicon. Contributes the ClickHouse
+ * applier (`clickhouseApply`, #3208), the Postgres applier (`postgresApply`,
+ * #3280) and their envelope projection (`toApplyResult`, one for both), which
+ * core's apply activity looks up by name for the `clickhouse` and `postgres`
+ * targets, the emulator lifecycles, and the steps of the two migration Ops.
+ */
+export { clickhouseApply, toApplyResult, resolveMarker } from "./clickhouse-apply";
+export type { ClickHouseApplyArgs, ClickHouseApplyDeps, ClickHouseApplyOutcome } from "./clickhouse-apply";
+export { clickhouseUp, clickhouseDown, CLICKHOUSE_EMULATOR, CLICKHOUSE_EMULATOR_SPEC } from "./clickhouse-emulator";
+export { postgresApply } from "./postgres-apply";
+export type { PostgresApplyArgs, PostgresApplyDeps, PostgresApplyOutcome } from "./postgres-apply";
+export { postgresUp, postgresDown, POSTGRES_EMULATOR, POSTGRES_EMULATOR_SPEC } from "./postgres-emulator";
+
+// The rebuild migration (#3198): one activity per step of ClickHouseRebuildOp
+// (../../clickhouse/rebuild/op.ts). Its receipts are kept in ClickHouse and
+// reached by the backfill directly, so this module exports no receiptRead or
+// receiptWrite: those names are global to a run and belong to the receipt row
+// of a project's aws or k8s lexicon.
+export {
+  clickhouseRebuildPlan,
+  clickhouseRebuildCreate,
+  clickhouseRebuildDualWrite,
+  clickhouseRebuildBackfill,
+  clickhouseRebuildVerify,
+  clickhouseRebuildSwap,
+  clickhouseRebuildRetain,
+  clickhouseRebuildDrop,
+  clickhouseRebuildCompensate,
+} from "./clickhouse-rebuild";
+export type { ClickHouseRebuildArgs, ClickHouseRebuildDeps, RebuildPlanResult } from "./clickhouse-rebuild";
+
+// The Postgres expand-and-contract migration (#3281): one activity per step of
+// PostgresMigrationOp (../../postgres/migrate/op.ts). Its receipts are kept in
+// the migrated table's schema and written by the backfill directly, in each
+// batch's own transaction.
+export {
+  postgresMigrationPlan,
+  postgresMigrationExpand,
+  postgresMigrationDualWrite,
+  postgresMigrationBackfill,
+  postgresMigrationCarry,
+  postgresMigrationVerify,
+  postgresMigrationSwitch,
+  postgresMigrationRetain,
+  postgresMigrationContract,
+  postgresMigrationCompensate,
+} from "./postgres-migration";
+export type { PostgresMigrationArgs, PostgresMigrationDeps, MigrationPlanResult } from "./postgres-migration";

package/src/op/activities/postgres-apply.e2e.test.ts

@@ -0,0 +1,184 @@
+/**
+ * The Postgres applier against the pinned server (#3280): create, then a
+ * column added with a default, an index built CONCURRENTLY on a table that
+ * exists, a foreign key added NOT VALID and then validated; an
+ * expand-and-contract change refused with the server untouched; a lock
+ * timeout while another session holds a lock on the table; the same build
+ * applied twice; a prune that drops an owned view and spares a table made by
+ * hand.
+ *
+ * Needs Docker; skips cleanly without it. One throwaway `postgres` container
+ * at the pin, removed afterwards even on failure.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { normalizeApply } from "@intentius/chant/apply";
+import { dockerAvailable, startTestPostgres, type TestPostgres } from "../../postgres/testing/server";
+import type { PostgresClient } from "../../postgres/live/client";
+import { planPgAgainstServer } from "../../postgres/plan/commands";
+import { PostgresApplyError } from "../../postgres/apply/apply";
+import { postgresApply, type PostgresApplyArgs, type PostgresApplyDeps } from "./postgres-apply";
+import { toApplyResult } from "./index";
+
+const enabled = await dockerAvailable();
+let server: TestPostgres | undefined;
+let admin: PostgresClient | undefined;
+const dir = mkdtempSync(join(tmpdir(), "chant-pg-apply-"));
+
+beforeAll(async () => {
+  if (!enabled) return;
+  server = await startTestPostgres();
+  admin = await server.connect();
+}, 600_000);
+
+afterAll(async () => {
+  await admin?.end();
+  await server?.stop();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+interface Obj {
+  export: string;
+  type: string;
+  ddl: string;
+  dependsOn?: string[];
+}
+
+let builds = 0;
+function buildFile(objects: Obj[]): string {
+  const path = join(dir, `schema-${builds++}.json`);
+  writeFileSync(path, JSON.stringify({ dialect: "postgres", postgresMajor: 18, applyOrder: objects.map((o) => o.export), objects: objects.map((o) => ({ dependsOn: [], ...o })) }));
+  return path;
+}
+
+const profile = () => ({
+  config: {
+    ownership: { stack: "shop", env: "e2e" },
+    sql: { profiles: { e2e: { url: server!.endpoint().url, password: { env: "PG_E2E_PASSWORD" } } } },
+  },
+  env: { PG_E2E_PASSWORD: server!.endpoint().password },
+});
+const deps = (): PostgresApplyDeps => ({ ...profile(), log: () => undefined });
+const apply = (objects: Obj[], args: Partial<PostgresApplyArgs> = {}) => postgresApply({ buildPath: buildFile(objects), environment: "e2e", ...args }, undefined, deps());
+
+const APP: Obj = { export: "app", type: "Postgres::Schema", ddl: "CREATE SCHEMA app;\nCOMMENT ON SCHEMA app IS 'The shop'" };
+const usersV1: Obj = {
+  export: "users",
+  type: "Postgres::Table",
+  dependsOn: ["app"],
+  ddl: "CREATE TABLE app.users (\n  id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,\n  email text NOT NULL\n);\nCOMMENT ON TABLE app.users IS 'One row per account'",
+};
+const ordersV1: Obj = {
+  export: "orders",
+  type: "Postgres::Table",
+  dependsOn: ["app"],
+  ddl: "CREATE TABLE app.orders (\n  id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,\n  user_id bigint NOT NULL,\n  amount numeric(12, 2) NOT NULL\n)",
+};
+const TOTALS: Obj = {
+  export: "totals",
+  type: "Postgres::View",
+  dependsOn: ["orders"],
+  ddl: "CREATE VIEW app.order_totals AS SELECT user_id, sum(amount) AS total FROM app.orders GROUP BY user_id",
+};
+const usersV2: Obj = {
+  ...usersV1,
+  ddl: usersV1.ddl.replace("email text NOT NULL", "email text NOT NULL,\n  status text NOT NULL DEFAULT 'active'"),
+};
+const ORDERS_USER_IDX: Obj = {
+  export: "ordersUser",
+  type: "Postgres::Index",
+  dependsOn: ["orders"],
+  ddl: "CREATE INDEX CONCURRENTLY orders_user_id_idx ON app.orders (user_id)",
+};
+const fk = (notValid: boolean): Obj => ({
+  ...ordersV1,
+  dependsOn: ["app", "users"],
+  ddl: ordersV1.ddl.replace(
+    "amount numeric(12, 2) NOT NULL\n)",
+    `amount numeric(12, 2) NOT NULL,\n  CONSTRAINT orders_user_fk FOREIGN KEY (user_id) REFERENCES app.users (id)${notValid ? " NOT VALID" : ""}\n)`,
+  ),
+});
+
+const one = async <T>(sql: string): Promise<T> => (await admin!.query<T>(sql))[0]!;
+
+describe.skipIf(!enabled)("applying to the pinned Postgres server", () => {
+  test("create, then each class of change, a refusal, a lock timeout, idempotence and prune", async () => {
+    // Create, and the server plans no changes afterwards.
+    const created = normalizeApply(toApplyResult(await apply([APP, usersV1, ordersV1, TOTALS])));
+    expect(created.applied.map((a) => [a.name, a.action])).toEqual([
+      ["app", "created"],
+      ["app.users", "created"],
+      ["app.orders", "created"],
+      ["app.order_totals", "created"],
+    ]);
+    expect(await one<{ c: string }>("SELECT obj_description('app.users'::regclass, 'pg_class') AS c")).toEqual({ c: "One row per account [chant managed-by=chant stack=shop env=e2e]" });
+    expect((await planPgAgainstServer("e2e", buildFile([APP, usersV1, ordersV1, TOTALS]), profile())).changes).toEqual([]);
+    await admin!.query("INSERT INTO app.users (email) VALUES ('a@example.com'), ('b@example.com')");
+    await admin!.query("INSERT INTO app.orders (user_id, amount) SELECT id, 10 FROM app.users");
+
+    // A column added with a default: one ALTER, existing rows read the default.
+    const added = await apply([APP, usersV2, ordersV1, TOTALS]);
+    expect(added.applied.find((a) => a.name === "app.users")).toMatchObject({
+      action: "updated",
+      statements: ["ALTER TABLE app.users ADD COLUMN status text DEFAULT 'active' NOT NULL"],
+    });
+    expect(await admin!.query("SELECT DISTINCT status FROM app.users")).toEqual([{ status: "active" }]);
+
+    // An index built CONCURRENTLY on a table that exists, outside any transaction.
+    const indexed = await apply([APP, usersV2, ordersV1, TOTALS, ORDERS_USER_IDX]);
+    expect(indexed.applied.find((a) => a.name === "app.orders_user_id_idx")?.action).toBe("created");
+    const cic = indexed.statements.find((s) => s.sql.startsWith("CREATE INDEX CONCURRENTLY"))!;
+    expect(cic).toMatchObject({ class: "concurrently", statementTimeoutMs: 0 });
+    expect(cic.transaction).toBeUndefined();
+    expect(await one("SELECT indisvalid FROM pg_index WHERE indexrelid = 'app.orders_user_id_idx'::regclass")).toEqual({ indisvalid: true });
+
+    // A foreign key added NOT VALID, then validated when the declaration drops NOT VALID.
+    await apply([APP, usersV2, fk(true), TOTALS, ORDERS_USER_IDX]);
+    expect(await one("SELECT convalidated FROM pg_constraint WHERE conname = 'orders_user_fk'")).toEqual({ convalidated: false });
+    const validated = await apply([APP, usersV2, fk(false), TOTALS, ORDERS_USER_IDX]);
+    expect(validated.statements.find((s) => s.object === "app.orders")).toMatchObject({ sql: "ALTER TABLE app.orders VALIDATE CONSTRAINT orders_user_fk", class: "validate" });
+    expect(await one("SELECT convalidated FROM pg_constraint WHERE conname = 'orders_user_fk'")).toEqual({ convalidated: true });
+    const FINAL = [APP, usersV2, fk(false), TOTALS, ORDERS_USER_IDX];
+
+    // An expand-and-contract change (a column rename) is refused and the server is untouched.
+    const renamed = { ...usersV2, ddl: usersV2.ddl.replace("email text NOT NULL,", "login text NOT NULL, -- previously: email") };
+    const refused = normalizeApply(toApplyResult(await apply([APP, renamed, fk(false), TOTALS, ORDERS_USER_IDX])));
+    expect(refused.notAttempted.find((n) => n.name === "app.users")).toMatchObject({ reason: "unsupported-kind" });
+    expect(refused.notAttempted.find((n) => n.name === "app.users")!.detail).toMatch(/SQLPG205.*PostgresMigrationOp\(\{ table: "app\.users", column: "login"/s);
+    expect(await one("SELECT count(*)::int AS n FROM information_schema.columns WHERE table_schema = 'app' AND table_name = 'users' AND column_name = 'email'")).toEqual({ n: 1 });
+
+    // A lock timeout: another session holds a lock on orders, and the ALTER gives up after lock_timeout instead of queueing.
+    const other = await server!.connect();
+    try {
+      await other.query("BEGIN");
+      await other.query("LOCK TABLE app.orders IN ACCESS SHARE MODE");
+      const withNote = { ...fk(false), ddl: fk(false).ddl.replace("amount numeric(12, 2) NOT NULL,", "amount numeric(12, 2) NOT NULL,\n  note text,") };
+      const started = Date.now();
+      const err = (await apply([APP, usersV2, withNote, TOTALS, ORDERS_USER_IDX], { lockTimeoutMs: 300 }).catch((e: unknown) => e)) as PostgresApplyError;
+      expect(Date.now() - started).toBeLessThan(10_000);
+      expect(err).toBeInstanceOf(PostgresApplyError);
+      expect(err.outcome.failed.map((f) => f.name)).toEqual(["app.orders"]);
+      expect(err.outcome.failed[0]!.error).toMatch(/lock timeout.*lock_timeout 300ms.*55P03/s);
+      expect(err.outcome.transactions.at(-1)).toMatchObject({ result: "rolled-back", failedAt: { sql: "ALTER TABLE app.orders ADD COLUMN note text" } });
+    } finally {
+      await other.query("ROLLBACK").catch(() => undefined);
+      await other.end();
+    }
+    expect(await one("SELECT count(*)::int AS n FROM information_schema.columns WHERE table_name = 'orders' AND column_name = 'note'")).toEqual({ n: 0 });
+
+    // The same build applied twice: the second sends nothing.
+    await apply(FINAL);
+    const again = await apply(FINAL);
+    expect(again.applied.map((a) => a.action)).toEqual(["unchanged", "unchanged", "unchanged", "unchanged", "unchanged"]);
+    expect(again.statements).toEqual([]);
+
+    // Prune drops the owned view the build no longer declares, and spares a table made by hand.
+    await admin!.query("CREATE TABLE app.handmade (id int)");
+    const pruned = await apply([APP, usersV2, fk(false), ORDERS_USER_IDX], { prune: true });
+    expect(pruned.pruned.map((p) => [p.kind, p.name, p.deleted])).toEqual([["Postgres::View", "app.order_totals", true]]);
+    expect(await one("SELECT to_regclass('app.order_totals') IS NULL AS gone, to_regclass('app.handmade') IS NOT NULL AS kept")).toEqual({ gone: true, kept: true });
+  }, 600_000);
+});