@intentius/chant-lexicon-sql 0.0.0-stage → 0.112.0

package/dist/validate-cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env tsx
+export {};
+//# sourceMappingURL=validate-cli.d.ts.map

package/dist/validate-cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validate-cli.d.ts","sourceRoot":"","sources":["../src/validate-cli.ts"],"names":[],"mappings":""}

package/dist/validate.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Validate the sql lexicon's generated artifacts.
+ *
+ * The core checks (the registry parses, `index.d.ts` exists and compiles), plus
+ * the ClickHouse catalog's own: the generated tables exist, the committed
+ * snapshot is at the pin, and the names a schema leans on hardest are still
+ * in it. A pin move that drops one of those would otherwise surface as lint
+ * going quiet, not as a failure.
+ */
+import { type ValidateResult } from "@intentius/chant/codegen/validate";
+export type { ValidateCheck, ValidateResult } from "@intentius/chant/codegen/validate";
+export declare function validate(opts?: {
+    basePath?: string;
+}): Promise<ValidateResult>;
+//# sourceMappingURL=validate.d.ts.map

package/dist/validate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../src/validate.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAKH,OAAO,EAAgD,KAAK,cAAc,EAAE,MAAM,mCAAmC,CAAC;AAMtH,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AA6DvF,wBAAsB,QAAQ,CAAC,IAAI,CAAC,EAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAAC,cAAc,CAAC,CAoEpF"}

package/package.json

@@ -1,6 +1,76 @@
 {
   "name": "@intentius/chant-lexicon-sql",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.112.0",
+  "description": "SQL lexicon for chant — database schema as typed declarations, one dialect at a time; ClickHouse first, with types read from a pinned server's catalog",
+  "license": "Apache-2.0",
+  "homepage": "https://intentius.io/chant",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/INTENTIUS/chant.git",
+    "directory": "lexicons/sql"
+  },
+  "bugs": {
+    "url": "https://github.com/INTENTIUS/chant/issues"
+  },
+  "keywords": [
+    "infrastructure-as-code",
+    "iac",
+    "typescript",
+    "sql",
+    "clickhouse",
+    "schema",
+    "chant"
+  ],
+  "type": "module",
+  "files": [
+    "src/",
+    "dist/"
+  ],
+  "exports": {
+    ".": {
+      "development": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "default": "./src/index.ts"
+    },
+    "./*": {
+      "development": "./src/*.ts",
+      "types": "./dist/*.d.ts",
+      "default": "./src/*.ts"
+    },
+    "./op/activities": {
+      "development": "./src/op/activities/index.ts",
+      "types": "./dist/op/activities/index.d.ts",
+      "default": "./src/op/activities/index.ts"
+    },
+    "./lint/post-synth": {
+      "development": "./src/lint/post-synth/index.ts",
+      "types": "./dist/lint/post-synth/index.d.ts",
+      "default": "./src/lint/post-synth/index.ts"
+    },
+    "./manifest": "./dist/manifest.json",
+    "./meta": "./dist/meta.json",
+    "./types": "./dist/types/index.d.ts"
+  },
+  "scripts": {
+    "generate": "tsx src/codegen/generate-cli.ts",
+    "bundle": "tsx src/package-cli.ts",
+    "validate": "tsx src/validate-cli.ts",
+    "docs": "tsx src/codegen/docs-cli.ts",
+    "prepack": "npm run generate && npm run bundle && npm run validate && npm run build",
+    "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && find dist -type f \\( -name \"*.js\" -o -name \"*.js.map\" \\) -delete"
+  },
+  "dependencies": {
+    "pg": "^8.23.1",
+    "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "@intentius/chant": "*",
+    "@types/pg": "^8.23.1",
+    "tsc-alias": "^1.8.17",
+    "typescript": "^5.9.3"
+  },
+  "peerDependencies": {
+    "@intentius/chant": "^0.112.0",
+    "typescript": "^5.9.3"
+  }
+}

package/src/clickhouse/apply/apply.ts

@@ -0,0 +1,288 @@
+/**
+ * The ClickHouse applier (#3208): make a server hold what a build declares,
+ * for every change the classifier (`../plan/rules.ts`) puts in the metadata
+ * or background-rewrite class, and refuse the rest.
+ *
+ * Per declared object, exactly one verdict (lexicon-authoring/applying.mdx):
+ *
+ * - APPLIED `created`: absent on the server; created from its declared
+ *   `CREATE`, with chant's ownership marker in its comment.
+ * - APPLIED `updated`: present, and the `ALTER`s for its changes ran. A
+ *   background rewrite (a column type change, a TTL change) is waited on in
+ *   `system.mutations` before the object counts as applied.
+ * - APPLIED `unchanged`: present, nothing to change, the marker in place.
+ * - NOT-ATTEMPTED `unsupported-kind`: a change ClickHouse cannot make with
+ *   `ALTER` (a sorting key, a partition key, an engine, a key column's type).
+ *   Nothing is sent for the object; the detail names each rule and the
+ *   restriction behind it, and names the rebuild migration Op to run instead
+ *   (`ClickHouseRebuildOp`, #3198).
+ * - NOT-ATTEMPTED `filtered`: the change drops a column, which destroys its
+ *   data, and the apply was not allowed to delete.
+ * - NOT-ATTEMPTED `dependency-failed`: an object it references is not on the
+ *   server, because creating it failed or was not attempted.
+ * - NOT-ATTEMPTED with the binding's reason: no server to apply to.
+ *
+ * A statement the server refuses is neither: the object was attempted and did
+ * not converge. The rest of the apply goes on, and the applier then throws
+ * {@link ClickHouseApplyError}, which carries the outcome so far.
+ *
+ * Prune, when asked for, drops the objects in the declared databases that the
+ * build no longer declares and whose comment carries this project's marker
+ * (stack and env). Nothing else is ever dropped: an object without the marker
+ * is not chant's, and one with another stack's marker is that stack's.
+ */
+
+import type { OwnershipMarker } from "@intentius/chant/ownership";
+import type { ApplyRef } from "@intentius/chant/apply";
+import {
+  SqlApplyError,
+  changesByObject,
+  dependencyFailedDetail,
+  missingDependencies,
+  readBuildObjects,
+  type SqlApplyOutcome,
+} from "../../core/apply";
+import { clickhouseQuery } from "../http";
+import type { ClickHouseTarget } from "../live/bind";
+import { readLiveSchema, type LiveObject } from "../live/catalog";
+import { canonicalObject, type CanonicalObject } from "../plan/normalize";
+import { diffSchemas, type Change } from "../plan/diff";
+import { dropFormattingOnly } from "../plan/server-format";
+import { CLICKHOUSE_ENTITY_TYPES, type ClickHouseEntityType } from "../entities";
+import { carriesMarker, isChantManaged } from "../ownership";
+import {
+  alterSteps,
+  commentStatement,
+  createStatement,
+  dropStatement,
+  isDestructiveAlter,
+  isRebuild,
+  refusalDetail,
+  sqlString,
+  stepsSetComment,
+  type DeclaredObject,
+  type Step,
+} from "./statements";
+import { waitForMutations } from "./mutations";
+
+export type { FailedObject } from "../../core/apply";
+
+/**
+ * The objects a `chant build` output declares, in creation order. Takes the
+ * sql lexicon's primary output, or a multi-lexicon output holding it under
+ * `sql` (the shared core's reader).
+ */
+export function declaredObjects(json: string, defaultDatabase = "default"): DeclaredObject[] {
+  return readBuildObjects(json, "clickhouse", (o) => {
+    const canonical = canonicalObject(o.ddl, defaultDatabase);
+    return {
+      exportName: o.export,
+      type: o.type as ClickHouseEntityType,
+      key: canonical.database !== undefined ? `${canonical.database}.${canonical.name}` : canonical.name,
+      ddl: o.ddl,
+      canonical,
+      dependsOn: o.dependsOn,
+    };
+  });
+}
+
+/** The plan axis: one ref per declared object, by its kind and its name on the server. */
+export const planRefs = (declared: readonly DeclaredObject[]): ApplyRef[] => declared.map((o) => ({ kind: o.type, name: o.key }));
+
+export interface ClickHouseApplyOptions {
+  /** The marker to stamp, and the identity a prune matches. Without a stack, prune declines. */
+  marker?: OwnershipMarker;
+  /** Drop owned objects the build no longer declares, and allow column drops. Default: off. */
+  prune?: boolean;
+  /** How long to wait for one table's mutations. Default: ten minutes. */
+  mutationTimeoutMs?: number;
+  signal?: AbortSignal;
+  /** Where to report each statement. Default: nowhere. */
+  log?: (line: string) => void;
+}
+
+/**
+ * The shared core's outcome: `target` is the server's URL, `source` where the
+ * binding came from (`sql.profiles.<env>` or `env CLICKHOUSE_URL`).
+ */
+export type ClickHouseApplyOutcome = SqlApplyOutcome;
+
+/** An apply in which the server refused a statement. The outcome says what did and did not happen. */
+export class ClickHouseApplyError extends SqlApplyError<ClickHouseApplyOutcome> {
+  constructor(outcome: ClickHouseApplyOutcome) {
+    super("ClickHouse", outcome);
+    this.name = "ClickHouseApplyError";
+  }
+}
+
+const liveKey = (o: LiveObject) => (o.database !== undefined ? `${o.database}.${o.name}` : o.name);
+
+/** Drop order: what reads from a table before the table, a database last. */
+const DROP_ORDER: Record<string, number> = {
+  [CLICKHOUSE_ENTITY_TYPES.materializedView]: 0,
+  [CLICKHOUSE_ENTITY_TYPES.view]: 1,
+  [CLICKHOUSE_ENTITY_TYPES.table]: 2,
+  [CLICKHOUSE_ENTITY_TYPES.database]: 3,
+};
+
+/**
+ * The server's objects in the databases the declarations use, as the diff
+ * compares them. The `default` database itself is never part of a schema.
+ */
+async function liveSchema(target: ClickHouseTarget, declared: readonly DeclaredObject[]): Promise<{ objects: LiveObject[]; canonical: Map<string, CanonicalObject> }> {
+  const databases = new Set(declared.map((o) => o.canonical.database ?? o.canonical.name));
+  const objects = (await readLiveSchema(target))
+    .filter((o) => !(o.type === CLICKHOUSE_ENTITY_TYPES.database && o.name === "default"))
+    .filter((o) => databases.has(o.database ?? o.name));
+  return { objects, canonical: new Map(objects.map((o) => [liveKey(o), canonicalObject(o.statement, target.defaultDatabase)])) };
+}
+
+/**
+ * The classified changes from the server to the declarations, keyed by the
+ * declared object (`database.name`), with formatting-only differences dropped
+ * by asking the server, as `chant sql plan` does.
+ */
+export async function plannedChanges(target: ClickHouseTarget, declared: readonly DeclaredObject[], live: Map<string, CanonicalObject>): Promise<Change[]> {
+  const diff = diffSchemas(
+    [...live].map(([key, canonical]) => ({ key, canonical })),
+    declared.map((o) => ({ key: o.key, canonical: o.canonical })),
+  );
+  return dropFormattingOnly(target.endpoint, diff.changes);
+}
+
+/**
+ * Apply the declarations to the server. Returns every declared object in
+ * exactly one of applied or not attempted, and every pruned orphan; throws
+ * {@link ClickHouseApplyError} when the server refused a statement.
+ */
+export async function applyClickHouse(target: ClickHouseTarget, declared: readonly DeclaredObject[], opts: ClickHouseApplyOptions = {}): Promise<ClickHouseApplyOutcome> {
+  const log = opts.log ?? (() => undefined);
+  const { objects: liveObjects, canonical: live } = await liveSchema(target, declared);
+  const liveByKey = new Map(liveObjects.map((o) => [liveKey(o), o]));
+  const changes = await plannedChanges(target, declared, live);
+  const byObject = changesByObject(changes);
+
+  const outcome: ClickHouseApplyOutcome = { target: target.endpoint.url, source: target.source, applied: [], pruned: [], notAttempted: [], failed: [] };
+  /** Export names whose object is on the server. */
+  const onServer = new Set<string>();
+  const exportNames = new Set(declared.map((o) => o.exportName));
+
+  const run = async (step: Step, obj: { database?: string; name: string }, ran: string[]) => {
+    opts.signal?.throwIfAborted();
+    log(step.sql);
+    await clickhouseQuery(target.endpoint, step.sql);
+    ran.push(step.sql);
+    if (step.rewrite && obj.database !== undefined) {
+      const ids = await waitForMutations(target.endpoint, obj.database, obj.name, {
+        ...(opts.mutationTimeoutMs !== undefined ? { timeoutMs: opts.mutationTimeoutMs } : {}),
+        ...(opts.signal ? { signal: opts.signal } : {}),
+      });
+      if (ids.length > 0) log(`-- waited for ${ids.join(", ")}`);
+    }
+  };
+
+  for (const obj of declared) {
+    const ref = { kind: obj.type, name: obj.key };
+    const mine = byObject.get(obj.key) ?? [];
+    const created = mine.some((c) => c.rule === "SQLCH200");
+    const renamedFrom = mine.find((c) => c.rule === "SQLCH230" || c.rule === "SQLCH231")?.before;
+    const liveObject = liveByKey.get(renamedFrom ?? obj.key);
+    const liveCanonical = live.get(renamedFrom ?? obj.key);
+
+    const refused = mine.filter(isRebuild);
+    if (refused.length > 0) {
+      if (liveObject) onServer.add(obj.exportName);
+      outcome.notAttempted.push({ ...ref, reason: "unsupported-kind", detail: refusalDetail(refused, obj.key, obj.type) });
+      continue;
+    }
+    const destructive = mine.filter(isDestructiveAlter);
+    if (destructive.length > 0 && !opts.prune) {
+      onServer.add(obj.exportName);
+      outcome.notAttempted.push({
+        ...ref,
+        reason: "filtered",
+        detail: `drops ${destructive.map((c) => c.field).join(", ")}, which destroys the data in it; an apply that may delete (prune, ApplyOp delete "owned-only" or "gated") makes it`,
+      });
+      continue;
+    }
+
+    const restamp = !created && !stepsSetComment(obj, mine) && (mine.some((c) => c.field === "comment") || !carriesMarker(liveObject?.comment, opts.marker));
+    const steps: Step[] = created
+      ? [{ sql: createStatement(obj, opts.marker), rewrite: false }]
+      : [...alterSteps(obj, mine, { ...(liveCanonical ? { live: liveCanonical } : {}), ...(opts.marker ? { marker: opts.marker } : {}) }), ...(restamp ? [{ sql: commentStatement(obj, opts.marker), rewrite: false }] : [])];
+
+    if (steps.length === 0) {
+      onServer.add(obj.exportName);
+      outcome.applied.push({ ...ref, action: "unchanged", ...(liveObject?.uuid ? { physicalId: liveObject.uuid } : {}), statements: [] });
+      continue;
+    }
+    const missing = missingDependencies(obj.exportName, obj.dependsOn, exportNames, onServer);
+    if (missing.length > 0) {
+      if (!created) onServer.add(obj.exportName);
+      outcome.notAttempted.push({ ...ref, reason: "dependency-failed", detail: dependencyFailedDetail(missing) });
+      continue;
+    }
+
+    const ran: string[] = [];
+    try {
+      for (const step of steps) await run(step, { ...(obj.canonical.database !== undefined ? { database: obj.canonical.database } : {}), name: obj.canonical.name }, ran);
+      onServer.add(obj.exportName);
+      outcome.applied.push({ ...ref, action: created ? "created" : "updated", ...(liveObject?.uuid ? { physicalId: liveObject.uuid } : {}), statements: ran });
+    } catch (err) {
+      if (opts.signal?.aborted) throw err;
+      if (!created) onServer.add(obj.exportName);
+      outcome.failed.push({ ...ref, error: err instanceof Error ? err.message : String(err), statements: ran });
+    }
+  }
+
+  if (opts.prune) await prune(target, changes, liveByKey, opts, outcome, log);
+  if (outcome.failed.length > 0) throw new ClickHouseApplyError(outcome);
+  return outcome;
+}
+
+async function prune(
+  target: ClickHouseTarget,
+  changes: readonly Change[],
+  liveByKey: Map<string, LiveObject>,
+  opts: ClickHouseApplyOptions,
+  outcome: ClickHouseApplyOutcome,
+  log: (line: string) => void,
+): Promise<void> {
+  const orphans = changes
+    .filter((c) => c.rule === "SQLCH250")
+    .map((c) => liveByKey.get(c.object))
+    .filter((o): o is LiveObject => o !== undefined)
+    .sort((a, b) => (DROP_ORDER[a.type] ?? 9) - (DROP_ORDER[b.type] ?? 9));
+
+  for (const o of orphans) {
+    const ref = { kind: o.type, name: liveKey(o) };
+    if (!isChantManaged(o.comment)) continue; // not chant's: never touched, never reported
+    if (!opts.marker?.stack) {
+      outcome.notAttempted.push({
+        ...ref,
+        reason: "not-prunable",
+        detail: "the project declares no ownership.stack, so this project's objects cannot be told from another chant project's",
+      });
+      continue;
+    }
+    if (!carriesMarker(o.comment, opts.marker)) continue; // another stack's or env's
+    try {
+      if (o.type === CLICKHOUSE_ENTITY_TYPES.database) {
+        const [row] = await clickhouseQuery<{ n: string | number }>(target.endpoint, `SELECT count() AS n FROM system.tables WHERE database = ${sqlString(o.name)}`);
+        const n = Number(row?.n ?? 0);
+        if (n > 0) {
+          outcome.notAttempted.push({ ...ref, reason: "not-prunable", detail: `still holds ${n} object(s) this apply does not drop; dropping the database would drop them` });
+          continue;
+        }
+      }
+      const sql = dropStatement(o.type, o.database, o.name);
+      opts.signal?.throwIfAborted();
+      log(sql);
+      await clickhouseQuery(target.endpoint, sql);
+      outcome.pruned.push({ ...ref, deleted: true, statement: sql });
+    } catch (err) {
+      if (opts.signal?.aborted) throw err;
+      outcome.failed.push({ ...ref, error: err instanceof Error ? err.message : String(err), statements: [] });
+    }
+  }
+}

package/src/clickhouse/apply/mutations.ts

@@ -0,0 +1,90 @@
+/**
+ * Waiting on a background rewrite (#3208).
+ *
+ * A column type change or a TTL change returns as soon as the server has
+ * recorded it; the rewrite of existing parts runs afterwards as a mutation,
+ * listed in `system.mutations` until it is done. An apply that reported
+ * APPLIED at that point would claim a table had converged while its parts
+ * were still being rewritten, so the applier waits for every unfinished
+ * mutation on the table, and reports the ids when it stops waiting.
+ */
+
+import { clickhouseQuery, type ClickHouseEndpoint } from "../http";
+import { sqlString } from "./statements";
+
+export interface PendingMutation {
+  mutation_id: string;
+  command: string;
+  latest_fail_reason: string;
+}
+
+/** A mutation that did not finish in time. The ids are what `system.mutations` and `KILL MUTATION` name. */
+export class MutationTimeoutError extends Error {
+  constructor(
+    readonly table: string,
+    readonly mutationIds: string[],
+    readonly timeoutMs: number,
+  ) {
+    super(
+      `${table}: mutation ${mutationIds.join(", ")} was still running after ${Math.round(timeoutMs / 1000)}s. ` +
+        `It goes on in the background; follow it in system.mutations, or stop it with KILL MUTATION WHERE mutation_id = '${mutationIds[0]}'.`,
+    );
+    this.name = "MutationTimeoutError";
+  }
+}
+
+/** A mutation the server keeps failing. It is retried forever, so waiting longer does not help. */
+export class MutationFailedError extends Error {
+  constructor(
+    readonly table: string,
+    readonly mutationId: string,
+    readonly reason: string,
+  ) {
+    super(
+      `${table}: mutation ${mutationId} is failing: ${reason.split("\n")[0]}. ` +
+        `The server retries it until it is killed: KILL MUTATION WHERE mutation_id = '${mutationId}'.`,
+    );
+    this.name = "MutationFailedError";
+  }
+}
+
+/** The table's mutations that are not done. */
+export async function pendingMutations(endpoint: ClickHouseEndpoint, database: string, table: string): Promise<PendingMutation[]> {
+  return clickhouseQuery<PendingMutation>(
+    endpoint,
+    `SELECT mutation_id, command, latest_fail_reason FROM system.mutations ` +
+      `WHERE database = ${sqlString(database)} AND table = ${sqlString(table)} AND NOT is_done ORDER BY create_time`,
+  );
+}
+
+export interface WaitOptions {
+  /** How long to wait in all. Default: ten minutes. */
+  timeoutMs?: number;
+  /** First poll interval; it doubles up to two seconds. Default: 100ms. */
+  intervalMs?: number;
+  signal?: AbortSignal;
+}
+
+/**
+ * Wait until the table has no unfinished mutation. Returns the ids waited on.
+ * Throws {@link MutationFailedError} as soon as one reports a failure, and
+ * {@link MutationTimeoutError} naming the ids still running at the deadline.
+ */
+export async function waitForMutations(endpoint: ClickHouseEndpoint, database: string, table: string, opts: WaitOptions = {}): Promise<string[]> {
+  const timeoutMs = opts.timeoutMs ?? 600_000;
+  const deadline = Date.now() + timeoutMs;
+  let interval = opts.intervalMs ?? 100;
+  const seen = new Set<string>();
+  const name = `${database}.${table}`;
+  for (;;) {
+    opts.signal?.throwIfAborted();
+    const pending = await pendingMutations(endpoint, database, table);
+    for (const m of pending) seen.add(m.mutation_id);
+    if (pending.length === 0) return [...seen];
+    const failing = pending.find((m) => m.latest_fail_reason);
+    if (failing) throw new MutationFailedError(name, failing.mutation_id, failing.latest_fail_reason);
+    if (Date.now() >= deadline) throw new MutationTimeoutError(name, pending.map((m) => m.mutation_id), timeoutMs);
+    await new Promise((r) => setTimeout(r, Math.min(interval, Math.max(0, deadline - Date.now()))));
+    interval = Math.min(interval * 2, 2_000);
+  }
+}