@intentius/chant-lexicon-sql 0.0.0-stage → 0.112.0

package/src/op/activities/postgres-migration.ts

@@ -0,0 +1,182 @@
+/**
+ * The expand-and-contract migration's activities (#3281), one per step of
+ * `PostgresMigrationOp` (`../../postgres/migrate/op.ts`). Each takes the same
+ * arguments, reads the build's declaration of the table and binds the
+ * environment's server the way the applier does (`./postgres-apply.ts`),
+ * opens one connection with node-postgres (loaded only now), runs its step
+ * (`../../postgres/migrate/`), which re-reads the server, and closes the
+ * connection.
+ */
+
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import type { ChantConfig } from "@intentius/chant/config";
+import { currentOpRun, parseDuration } from "@intentius/chant/op";
+import { resolveOwnershipMarker } from "../../core/apply";
+import { resolveBoundTarget } from "../../postgres/live/bind";
+import { connectPostgres, type PostgresClient, type PostgresEndpoint } from "../../postgres/live/client";
+import { DEFAULT_POSTGRES_APPLY_TIMEOUTS, declaredObjects } from "../../postgres/apply/apply";
+import { PG_CLASSIFIER_RULES } from "../../postgres/plan/rules";
+import { declaredTable, migrationPlanDigest, observeMigration } from "../../postgres/migrate/observe";
+import { backfill, verifyMigration, type BackfillDeps, type BackfillResult, type VerifyResult } from "../../postgres/migrate/backfill";
+import {
+  carryOver,
+  compensate,
+  contract,
+  expand,
+  retainPlan,
+  startDualWrite,
+  switchColumns,
+  type CarryResult,
+  type CompensateResult,
+  type ContractResult,
+  type DualWriteResult,
+  type ExpandResult,
+  type MigrationRun,
+  type RetainResult,
+  type SwitchResult,
+} from "../../postgres/migrate/steps";
+import { DEFAULT_REPLICATION_LAG } from "../../postgres/migrate/replication";
+import type { PostgresMigrationArgs } from "../../postgres/migrate/op";
+import { buildMajor } from "./postgres-apply";
+
+export type { PostgresMigrationArgs } from "../../postgres/migrate/op";
+
+/** What a test (or an embedding caller) may inject instead of the project's config, the process env and node-postgres. */
+export interface PostgresMigrationDeps {
+  config?: Pick<ChantConfig, "sql" | "ownership">;
+  env?: Record<string, string | undefined>;
+  log?: (line: string) => void;
+  connect?: (endpoint: PostgresEndpoint) => Promise<PostgresClient>;
+  backfill?: BackfillDeps;
+}
+
+async function loadConfig(cwd: string): Promise<Pick<ChantConfig, "sql" | "ownership"> | undefined> {
+  try {
+    const { loadChantConfig } = await import("@intentius/chant/config");
+    return (await loadChantConfig(cwd)).config;
+  } catch {
+    return undefined;
+  }
+}
+
+/** Bind the server, connect and read the declaration, run `body` with it, then close the connection. */
+async function withRun<T>(args: PostgresMigrationArgs, signal: AbortSignal | undefined, deps: PostgresMigrationDeps, body: (run: MigrationRun) => Promise<T>): Promise<T> {
+  const cwd = args.cwd ?? process.cwd();
+  const config = deps.config ?? (await loadConfig(cwd));
+  const marker = resolveOwnershipMarker(args, config, "PostgresMigrationOp");
+  const target = await resolveBoundTarget({ ...(args.environment !== undefined ? { environment: args.environment } : {}), config: config ?? {}, ...(deps.env ? { env: deps.env } : {}) });
+  const json = readFileSync(resolve(cwd, args.buildPath), "utf8");
+  const objects = declaredObjects(json, target.defaultSchema);
+  const declared = declaredTable(objects, args.table);
+  const major = buildMajor(json) ?? config?.sql?.postgresMajor;
+  const lag = args.replicationLag;
+  const client = await (deps.connect ?? ((e) => connectPostgres(e, { applicationName: "chant migration" })))(target.endpoint);
+  const runId = currentOpRun()?.runId;
+  try {
+    return await body({
+      client,
+      target,
+      declared,
+      objects,
+      column: args.column,
+      ...(args.using !== undefined ? { using: args.using } : {}),
+      ...(marker ? { marker } : {}),
+      ...(major !== undefined ? { major } : {}),
+      batchSize: args.batchSize ?? 1000,
+      retainMs: parseDuration(args.retain ?? "7d"),
+      replicationLag:
+        lag === false
+          ? false
+          : {
+              maxLagMs: lag?.max !== undefined ? parseDuration(lag.max) : DEFAULT_REPLICATION_LAG.maxLagMs,
+              waitMs: lag?.wait !== undefined ? parseDuration(lag.wait) : DEFAULT_REPLICATION_LAG.waitMs,
+            },
+      lockTimeoutMs: args.lockTimeoutMs ?? target.timeouts?.lockTimeoutMs ?? DEFAULT_POSTGRES_APPLY_TIMEOUTS.lockTimeoutMs,
+      statementTimeoutMs: args.statementTimeoutMs ?? target.timeouts?.statementTimeoutMs ?? DEFAULT_POSTGRES_APPLY_TIMEOUTS.statementTimeoutMs,
+      log: deps.log ?? ((line: string) => console.log(line)),
+      ...(signal ? { signal } : {}),
+      ...(runId ? { runId } : {}),
+    });
+  } finally {
+    await client.end();
+  }
+}
+
+export interface MigrationPlanResult {
+  state: "migrate" | "switched" | "done";
+  /** `schema.table.column`. */
+  migration: string;
+  change: "rename" | "type";
+  /** The plan's digest, before the verification; present while there is a migration to run. */
+  planDigest?: string;
+  /** Each classified change, as `<rule> <field>: <before> -> <after>`. */
+  changes: string[];
+  summary: string;
+}
+
+/** Plan: classify the column's change against the server and say how far the migration has got. */
+export async function postgresMigrationPlan(args: PostgresMigrationArgs, signal?: AbortSignal, deps: PostgresMigrationDeps = {}): Promise<MigrationPlanResult> {
+  return withRun(args, signal, deps, async (run) => {
+    const o = await observeMigration({
+      client: run.client,
+      target: run.target,
+      declared: run.declared,
+      ...(run.objects ? { objects: run.objects } : {}),
+      column: run.column,
+      ...(run.marker ? { marker: run.marker } : {}),
+      ...(run.using !== undefined ? { using: run.using } : {}),
+      ...(run.major !== undefined ? { major: run.major } : {}),
+    });
+    const n = o.names;
+    const changes = o.changes.map((c) => `${c.rule} ${c.field}${c.before !== undefined || c.after !== undefined ? `: ${c.before ?? "none"} -> ${c.after ?? "none"}` : ""}`);
+    const made = o.changes.filter((c) => c.rule === "SQLPG205" || c.rule === "SQLPG207" || c.rule === "SQLPG208").map((c) => `${c.rule} ${PG_CLASSIFIER_RULES[c.rule].title}`);
+    const summary =
+      o.state === "migrate"
+        ? `${n.key} needs expand and contract (${made.join("; ")}): ${n.change === "rename" ? `${n.source} renamed to ${n.column}` : `${n.column} as ${o.column.type}, computed as ${o.expression}`}, batched by ${o.batchKey}${o.newColumn ? `; ${n.newColumn} already added` : ""}`
+        : o.state === "switched"
+          ? `${n.key} has been switched; the old column is kept as ${n.oldColumn}`
+          : `${n.key} holds its declaration; nothing to migrate`;
+    run.log(`-- ${summary}`);
+    for (const c of changes) run.log(`--   ${c}`);
+    return { state: o.state, migration: n.key, change: n.change, ...(o.state === "migrate" ? { planDigest: migrationPlanDigest(o) } : {}), changes, summary };
+  });
+}
+
+export async function postgresMigrationExpand(args: PostgresMigrationArgs, signal?: AbortSignal, deps: PostgresMigrationDeps = {}): Promise<ExpandResult> {
+  return withRun(args, signal, deps, expand);
+}
+
+export async function postgresMigrationDualWrite(args: PostgresMigrationArgs, signal?: AbortSignal, deps: PostgresMigrationDeps = {}): Promise<DualWriteResult> {
+  return withRun(args, signal, deps, startDualWrite);
+}
+
+export async function postgresMigrationBackfill(args: PostgresMigrationArgs, signal?: AbortSignal, deps: PostgresMigrationDeps = {}): Promise<BackfillResult> {
+  return withRun(args, signal, deps, (run) => backfill(run, deps.backfill));
+}
+
+/** Carry over: the indexes and constraints on the old column, made again on the new one. */
+export async function postgresMigrationCarry(args: PostgresMigrationArgs, signal?: AbortSignal, deps: PostgresMigrationDeps = {}): Promise<CarryResult> {
+  return withRun(args, signal, deps, carryOver);
+}
+
+export async function postgresMigrationVerify(args: PostgresMigrationArgs, signal?: AbortSignal, deps: PostgresMigrationDeps = {}): Promise<VerifyResult> {
+  return withRun(args, signal, deps, verifyMigration);
+}
+
+export async function postgresMigrationSwitch(args: PostgresMigrationArgs, signal?: AbortSignal, deps: PostgresMigrationDeps = {}): Promise<SwitchResult> {
+  return withRun(args, signal, deps, switchColumns);
+}
+
+export async function postgresMigrationRetain(args: PostgresMigrationArgs, signal?: AbortSignal, deps: PostgresMigrationDeps = {}): Promise<RetainResult> {
+  return withRun(args, signal, deps, retainPlan);
+}
+
+export async function postgresMigrationContract(args: PostgresMigrationArgs, signal?: AbortSignal, deps: PostgresMigrationDeps = {}): Promise<ContractResult> {
+  return withRun(args, signal, deps, contract);
+}
+
+/** onFailure: drop what the expand added. */
+export async function postgresMigrationCompensate(args: PostgresMigrationArgs, signal?: AbortSignal, deps: PostgresMigrationDeps = {}): Promise<CompensateResult> {
+  return withRun(args, signal, deps, compensate);
+}

package/src/op/activity-contracts.ts

@@ -0,0 +1,210 @@
+/**
+ * Activity contracts for the sql lexicon's own `op/activities` (chant #2101):
+ * the ClickHouse rebuild migration's steps (#3198) and the Postgres
+ * expand-and-contract migration's (#3281).
+ *
+ * `loadActivityContracts` imports this module at
+ * `@intentius/chant-lexicon-sql/op/activity-contracts` for every project that
+ * lists the sql lexicon, and core's OPS012 and OPS013 check every
+ * `ClickHouseRebuildOp` and `PostgresMigrationOp` step against it at
+ * `chant build`: the arguments, and the two references each Op makes into a
+ * step's return value, the verification's `planDigest` (the swap or switch
+ * gate) and the retention's `dropDigest` or `contractDigest` (the drop or
+ * contract gate). Arguments are `z.strictObject`, so a
+ * misspelled key fails the build instead of vanishing.
+ */
+
+import { z } from "zod";
+import { activityContract } from "@intentius/chant/op";
+
+const dualWrite = z.discriminatedUnion("mode", [
+  z.strictObject({ mode: z.literal("materialized-view"), cutoverColumn: z.string(), cutoverDelay: z.string().optional() }),
+  z.strictObject({ mode: z.literal("app") }),
+]);
+
+/** `ClickHouseRebuildArgs` (`../clickhouse/rebuild/op.ts`): the same for every step. */
+const rebuildArgs = z.strictObject({
+  table: z.string(),
+  buildPath: z.string(),
+  environment: z.string().optional(),
+  dualWrite,
+  retain: z.string().optional(),
+  mutationTimeout: z.string().optional(),
+  replicaTimeout: z.string().optional(),
+  stack: z.string().optional(),
+  ownershipEnv: z.string().optional(),
+  cwd: z.string().optional(),
+});
+
+const state = z.enum(["rebuild", "swapped", "done"]);
+const tableEntity = { entities: ["table"] };
+
+export const clickhouseRebuildPlanContract = activityContract(
+  "clickhouseRebuildPlan",
+  rebuildArgs,
+  z.object({ state, table: z.string(), planDigest: z.string().optional(), changes: z.array(z.string()), summary: z.string() }),
+  tableEntity,
+);
+
+export const clickhouseRebuildCreateContract = activityContract(
+  "clickhouseRebuildCreate",
+  rebuildArgs,
+  z.object({ state, table: z.string(), created: z.boolean() }),
+  tableEntity,
+);
+
+export const clickhouseRebuildDualWriteContract = activityContract(
+  "clickhouseRebuildDualWrite",
+  rebuildArgs,
+  z.object({ state, mode: z.enum(["materialized-view", "app"]), cutover: z.string().optional(), created: z.boolean() }),
+  tableEntity,
+);
+
+export const clickhouseRebuildBackfillContract = activityContract(
+  "clickhouseRebuildBackfill",
+  rebuildArgs,
+  z.object({ state, partitions: z.number(), copied: z.number(), skipped: z.number(), cleared: z.number() }),
+  tableEntity,
+);
+
+export const clickhouseRebuildVerifyContract = activityContract(
+  "clickhouseRebuildVerify",
+  rebuildArgs,
+  z.object({
+    state,
+    planDigest: z.string().optional(),
+    partitions: z.number(),
+    rows: z.number(),
+    summary: z.string(),
+    verification: z.array(z.object({ partition: z.string(), rows: z.number(), checksum: z.string() })),
+  }),
+  tableEntity,
+);
+
+export const clickhouseRebuildSwapContract = activityContract(
+  "clickhouseRebuildSwap",
+  rebuildArgs,
+  z.object({ state, swapped: z.boolean(), dependents: z.array(z.string()), oldTable: z.string().optional(), retainUntil: z.string().optional() }),
+  tableEntity,
+);
+
+export const clickhouseRebuildRetainContract = activityContract(
+  "clickhouseRebuildRetain",
+  rebuildArgs,
+  z.object({ state, oldTable: z.string().optional(), retainUntil: z.string().optional(), due: z.boolean(), dropDigest: z.string().optional() }),
+  tableEntity,
+);
+
+export const clickhouseRebuildDropContract = activityContract(
+  "clickhouseRebuildDrop",
+  rebuildArgs,
+  z.object({ state, dropped: z.boolean(), oldTable: z.string().optional(), retainUntil: z.string().optional() }),
+  tableEntity,
+);
+
+export const clickhouseRebuildCompensateContract = activityContract(
+  "clickhouseRebuildCompensate",
+  rebuildArgs,
+  z.object({ dropped: z.array(z.string()) }),
+  tableEntity,
+);
+
+// ── PostgresMigrationOp (#3281) ────────────────────────────────────────
+
+/** `PostgresMigrationArgs` (`../postgres/migrate/op.ts`): the same for every step. */
+const migrationArgs = z.strictObject({
+  table: z.string(),
+  column: z.string(),
+  buildPath: z.string(),
+  environment: z.string().optional(),
+  using: z.string().optional(),
+  retain: z.string().optional(),
+  batchSize: z.number().int().positive().optional(),
+  replicationLag: z.union([z.literal(false), z.strictObject({ max: z.string().optional(), wait: z.string().optional() })]).optional(),
+  lockTimeoutMs: z.number().int().nonnegative().optional(),
+  statementTimeoutMs: z.number().int().nonnegative().optional(),
+  stack: z.string().optional(),
+  ownershipEnv: z.string().optional(),
+  cwd: z.string().optional(),
+});
+
+const migrationState = z.enum(["migrate", "switched", "done"]);
+
+export const postgresMigrationPlanContract = activityContract(
+  "postgresMigrationPlan",
+  migrationArgs,
+  z.object({ state: migrationState, migration: z.string(), change: z.enum(["rename", "type"]), planDigest: z.string().optional(), changes: z.array(z.string()), summary: z.string() }),
+  tableEntity,
+);
+
+export const postgresMigrationExpandContract = activityContract(
+  "postgresMigrationExpand",
+  migrationArgs,
+  z.object({ state: migrationState, column: z.string(), added: z.boolean() }),
+  tableEntity,
+);
+
+export const postgresMigrationDualWriteContract = activityContract(
+  "postgresMigrationDualWrite",
+  migrationArgs,
+  z.object({ state: migrationState, trigger: z.string(), created: z.boolean() }),
+  tableEntity,
+);
+
+export const postgresMigrationBackfillContract = activityContract(
+  "postgresMigrationBackfill",
+  migrationArgs,
+  z.object({ state: migrationState, batches: z.number(), filled: z.number(), skipped: z.number(), rows: z.number(), pausedMs: z.number() }),
+  tableEntity,
+);
+
+export const postgresMigrationCarryContract = activityContract(
+  "postgresMigrationCarry",
+  migrationArgs,
+  z.object({ state: migrationState, built: z.number(), ready: z.number(), views: z.number(), carried: z.array(z.string()) }),
+  tableEntity,
+);
+
+export const postgresMigrationVerifyContract = activityContract(
+  "postgresMigrationVerify",
+  migrationArgs,
+  z.object({
+    state: migrationState,
+    planDigest: z.string().optional(),
+    rows: z.number(),
+    mismatched: z.number(),
+    nulls: z.number(),
+    checksum: z.string(),
+    expected: z.string(),
+    summary: z.string(),
+  }),
+  tableEntity,
+);
+
+export const postgresMigrationSwitchContract = activityContract(
+  "postgresMigrationSwitch",
+  migrationArgs,
+  z.object({ state: migrationState, switched: z.boolean(), oldColumn: z.string().optional(), retainUntil: z.string().optional() }),
+  tableEntity,
+);
+
+export const postgresMigrationRetainContract = activityContract(
+  "postgresMigrationRetain",
+  migrationArgs,
+  z.object({ state: migrationState, oldColumn: z.string().optional(), retainUntil: z.string().optional(), due: z.boolean(), contractDigest: z.string().optional() }),
+  tableEntity,
+);
+
+export const postgresMigrationContractContract = activityContract(
+  "postgresMigrationContract",
+  migrationArgs,
+  z.object({ state: migrationState, dropped: z.boolean(), oldColumn: z.string().optional(), retainUntil: z.string().optional() }),
+  tableEntity,
+);
+
+export const postgresMigrationCompensateContract = activityContract(
+  "postgresMigrationCompensate",
+  migrationArgs,
+  z.object({ dropped: z.array(z.string()) }),
+  tableEntity,
+);

package/src/package-cli.ts

@@ -0,0 +1,17 @@
+#!/usr/bin/env tsx
+/**
+ * Thin entry point for `npm run bundle` in lexicon-sql.
+ */
+import { join, dirname } from "path";
+import { fileURLToPath } from "url";
+import { writeBundleSpec } from "@intentius/chant/codegen/package";
+import { generate, writeGeneratedFiles } from "./codegen/generate";
+import { packageLexicon } from "./codegen/package";
+
+const pkgDir = dirname(dirname(fileURLToPath(import.meta.url)));
+
+writeGeneratedFiles(await generate({ verbose: true }));
+const { spec, stats } = await packageLexicon({ verbose: true });
+writeBundleSpec(spec, join(pkgDir, "dist"));
+
+console.error(`Packaged ${stats.ruleCount} rules, ${stats.skillCount} skills`);

package/src/plugin.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, test } from "vitest";
+import { isLexiconPlugin } from "@intentius/chant/lexicon";
+import { sqlPlugin } from "./plugin";
+import { sqlConfigSchema } from "./config";
+import { CLICKHOUSE_VERSION } from "./spec/pin";
+
+describe("sql plugin", () => {
+  test("is a LexiconPlugin named sql", () => {
+    expect(isLexiconPlugin(sqlPlugin)).toBe(true);
+    expect(sqlPlugin.name).toBe("sql");
+  });
+
+  test("carries the sql serializer and its rule prefix", () => {
+    expect(sqlPlugin.serializer.name).toBe("sql");
+    expect(sqlPlugin.serializer.rulePrefix).toBe("SQL");
+  });
+
+  test("registers the LSP providers and docs()", () => {
+    expect(typeof sqlPlugin.completionProvider).toBe("function");
+    expect(typeof sqlPlugin.hoverProvider).toBe("function");
+    expect(typeof sqlPlugin.docs).toBe("function");
+  });
+
+  test("every rule and check id it ships starts with SQL", () => {
+    const ids = [...(sqlPlugin.lintRules?.() ?? []).map((r) => r.id), ...(sqlPlugin.postSynthChecks?.() ?? []).map((c) => c.id)];
+    for (const id of ids) expect(id.startsWith("SQL")).toBe(true);
+  });
+});
+
+describe("the sql config namespace", () => {
+  test("accepts a known dialect and an empty namespace", () => {
+    expect(sqlConfigSchema.safeParse({ dialect: "clickhouse" }).success).toBe(true);
+    expect(sqlConfigSchema.safeParse({ dialect: "postgres" }).success).toBe(true);
+    expect(sqlConfigSchema.safeParse({}).success).toBe(true);
+    for (const provider of ["rds", "aurora", "cloud-sql", "azure", "neon", "supabase"]) {
+      expect(sqlConfigSchema.safeParse({ dialect: "postgres", provider }).success, provider).toBe(true);
+      expect(sqlConfigSchema.safeParse({ profiles: { prod: { url: "postgres://x", provider } } }).success, provider).toBe(true);
+    }
+    expect(sqlConfigSchema.safeParse({ provider: "heroku" }).success).toBe(false);
+  });
+
+  test("accepts a list of dialects", () => {
+    expect(sqlConfigSchema.safeParse({ dialect: ["clickhouse"] }).success).toBe(true);
+    expect(sqlConfigSchema.safeParse({ dialect: ["clickhouse", "postgres"] }).success).toBe(true);
+    expect(sqlConfigSchema.safeParse({ dialect: [] }).success).toBe(false);
+  });
+
+  test("refuses any other name as unknown", () => {
+    const message = (dialect: unknown) => sqlConfigSchema.safeParse({ dialect }).error?.issues[0]?.message;
+    expect(message("mysql")).toBe("expected a dialect (clickhouse, postgres) or a non-empty list of them");
+    expect(message(["postgres", "mysql"])).toBe("expected a dialect (clickhouse, postgres) or a non-empty list of them");
+  });
+
+  test("refuses an unknown key rather than ignoring it", () => {
+    expect(sqlConfigSchema.safeParse({ dialects: "clickhouse" }).success).toBe(false);
+  });
+});
+
+describe("the upstream pin", () => {
+  const pin = sqlPlugin.upstreamPin!;
+  const line = `export const CLICKHOUSE_VERSION = "${CLICKHOUSE_VERSION}";`;
+
+  test("reads the version constant", () => {
+    expect(pin.pattern.exec(line)?.[1]).toBe(CLICKHOUSE_VERSION);
+  });
+
+  test("writes a GitHub LTS tag back as the bare version the docker tag uses", () => {
+    expect(pin.replace("v26.8.16.2-lts", line)).toBe('export const CLICKHOUSE_VERSION = "26.8.16.2";');
+  });
+
+  test("looks only at LTS releases of ClickHouse", () => {
+    expect(pin.upstream).toEqual({ owner: "ClickHouse", repo: "ClickHouse", kind: "releases", tagSuffix: "-lts" });
+  });
+});