@intentius/chant-lexicon-sql 0.116.0 → 0.118.0

package/src/clickhouse/rebuild/rebuild.test.ts

@@ -21,6 +21,7 @@ import * as activitiesModule from "../../op/activities";
 import { ClickHouseRebuildOp, type ClickHouseRebuildOpConfig } from "./op";
 import { copiedColumns } from "./observe";
 import { partitionEffect } from "./backfill";
+import { insertTarget, waitForCutover, type CutoverProbe } from "./steps";
 import { intoShard, onShard, onShardServers, shardTable } from "./shards";
 import { renameColumns, sourcePartitionExpression } from "./partitions";
 import { canonicalObject } from "../plan/normalize";
@@ -240,3 +241,56 @@ describe("a rebuild across the shards of a cluster (#3663)", () => {
     expect(onShardServers(shard2)).toBe("getMacro('shard') = 's2'");
   });
 });
+
+describe("the wait for the cut-over (#3700)", () => {
+  const CUT = Date.parse("2026-10-10T12:00:05Z");
+  /** A probe on a fake clock that starts at `start` and moves with real time, and writes that finish after `writesFor` polls. */
+  function probe(start: number, writesFor: number): CutoverProbe & { polls: number } {
+    const t0 = Date.now();
+    const p = {
+      polls: 0,
+      now: async () => start + (Date.now() - t0),
+      pendingWrites: async (cutover: number) => {
+        expect(cutover).toBe(CUT);
+        p.polls++;
+        return p.polls <= writesFor ? ["insert-1"] : [];
+      },
+    };
+    return p;
+  }
+  const run = (lines: string[] = []) => ({ log: (l: string) => void lines.push(l) });
+
+  test("it ends as soon as the clock has passed the cut-over and no write begun before it is running", async () => {
+    const lines: string[] = [];
+    const p = probe(CUT - 300, 2);
+    const began = Date.now();
+    await waitForCutover(run(lines), CUT, p, { pollMs: 10 });
+    expect(Date.now() - began).toBeLessThan(2_000);
+    expect(p.polls).toBe(3);
+    expect(lines[0]).toMatch(/^-- waiting 1s for the cut-over at 2026-10-10T12:00:05.000Z/);
+    expect(lines.filter((l) => l.includes("insert-1"))).toHaveLength(1);
+  });
+
+  test("a quiet table past its cut-over does not wait at all", async () => {
+    const p = probe(CUT + 1, 0);
+    const began = Date.now();
+    await waitForCutover(run(), CUT, p);
+    expect(Date.now() - began).toBeLessThan(100);
+    expect(p.polls).toBe(1);
+  });
+
+  test("a write still running at the timeout stops the step, naming it", async () => {
+    await expect(waitForCutover(run(), CUT, probe(CUT + 1, Infinity), { timeoutMs: 50, pollMs: 10 })).rejects.toThrow(
+      /begun before the cut-over at 2026-10-10T12:00:05.000Z are still running after 0s: insert-1\. .*cutoverTimeout/,
+    );
+  });
+
+  test("the table an INSERT writes to", () => {
+    expect(insertTarget("INSERT INTO shop.events VALUES (1)", "default")).toEqual({ database: "shop", name: "events" });
+    expect(insertTarget("  /* app */ insert into `shop`.`events` (ts) FORMAT JSONEachRow", "default")).toEqual({ database: "shop", name: "events" });
+    expect(insertTarget("INSERT INTO TABLE events SELECT 1", "shop")).toEqual({ database: "shop", name: "events" });
+    expect(insertTarget('INSERT INTO "shop" . "events" VALUES', "default")).toEqual({ database: "shop", name: "events" });
+    expect(insertTarget("INSERT INTO FUNCTION remote('h', shop.events) VALUES (1)", "shop")).toBeUndefined();
+    expect(insertTarget("SELECT * FROM shop.events", "shop")).toBeUndefined();
+  });
+});

package/src/clickhouse/rebuild/steps.ts

@@ -42,6 +42,8 @@ export interface RebuildRun {
   mutationTimeoutMs?: number;
   /** In a Replicated database, how long a step waits for this replica to fetch what the others wrote. Default: two minutes. */
   replicaTimeoutMs?: number;
+  /** How long the backfill waits, once the cut-over has passed, for writes into the old table begun before it. Default: ten minutes. */
+  cutoverTimeoutMs?: number;
   log: (line: string) => void;
   signal?: AbortSignal;
   /** The run's id, recorded on the receipts it writes. */
@@ -181,7 +183,7 @@ export async function startDualWrite(run: RebuildRun): Promise<DualWriteResult>
   // On a whole second: a DateTime column holds whole seconds, and a row written
   // during the cut-over's second would otherwise carry a time before it and
   // reach the old table after the backfill has started.
-  const cutover = Math.ceil(((await serverNow(run.target)) + parseDuration(run.dualWrite.cutoverDelay ?? "1m")) / 1000) * 1000;
+  const cutover = Math.ceil(((await serverNow(run.target)) + parseDuration(run.dualWrite.cutoverDelay ?? CUTOVER_DELAY)) / 1000) * 1000;
   const iso = new Date(cutover).toISOString();
   const select = o.copied.map((c) => (c.source === c.name ? ident(c.name) : `${ident(c.source)} AS ${ident(c.name)}`)).join(", ");
   const comment = stampedComment(`chant rebuild of ${o.names.key}: rows at or after the cut-over`, run.marker, working(o.names.key, "dual", { cutover: iso }));
@@ -193,6 +195,9 @@ export async function startDualWrite(run: RebuildRun): Promise<DualWriteResult>
   return { state: o.state, mode, cutover: iso, created: true };
 }
 
+/** How far after the dual-write view is made the cut-over falls, unless `cutoverDelay` says otherwise. */
+export const CUTOVER_DELAY = "5s";
+
 /** The cut-over in milliseconds, from the dual-write view; undefined in app mode. */
 export function cutoverOf(run: RebuildRun, o: RebuildObservation): number | undefined {
   if (run.dualWrite.mode === "app") return undefined;
@@ -201,6 +206,136 @@ export function cutoverOf(run: RebuildRun, o: RebuildObservation): number | unde
   return Date.parse(iso);
 }
 
+/**
+ * What the backfill waits on before it reads the rows before the cut-over.
+ * The default reads the server (`serverCutoverProbe`); a test passes its own.
+ */
+export interface CutoverProbe {
+  /** The server's clock, in milliseconds. */
+  now(): Promise<number>;
+  /**
+   * The writes into the old table begun before `cutover` that have not
+   * finished: INSERTs still running, and asynchronous inserts still queued.
+   * Each is named by its query id.
+   */
+  pendingWrites(cutover: number): Promise<string[]>;
+}
+
+/** How long the backfill waits, once the cut-over has passed, for the writes begun before it. Default: ten minutes. */
+export const CUTOVER_TIMEOUT_MS = 10 * 60_000;
+
+/**
+ * Wait until no row before the cut-over is still to arrive in the old table,
+ * then return. Two conditions, both polled: the server's clock has passed
+ * the cut-over (until then a row can still be written with a time before
+ * it), and every write into the old table that began before the cut-over has
+ * finished (a long INSERT, or an asynchronous insert still in its buffer,
+ * can carry such rows after the clock has passed). The second wait is
+ * bounded by `timeoutMs`; past it the step stops naming the writes, rather
+ * than copy without their rows. A row whose time lags further behind its
+ * write than `cutoverDelay` is neither: the verification finds it.
+ */
+export async function waitForCutover(
+  run: Pick<RebuildRun, "log" | "signal">,
+  cutover: number,
+  probe: CutoverProbe,
+  opts: { timeoutMs?: number; pollMs?: number } = {},
+): Promise<void> {
+  const at = new Date(cutover).toISOString();
+  for (;;) {
+    const now = await probe.now();
+    if (now > cutover) break;
+    run.log(`-- waiting ${Math.ceil((cutover - now) / 1000)}s for the cut-over at ${at}`);
+    await sleep(Math.min(cutover - now + 50, 10_000), run.signal);
+  }
+  const timeoutMs = opts.timeoutMs ?? CUTOVER_TIMEOUT_MS;
+  const deadline = Date.now() + timeoutMs;
+  let pollMs = opts.pollMs ?? 100;
+  let logged = "";
+  for (;;) {
+    const pending = await probe.pendingWrites(cutover);
+    if (pending.length === 0) return;
+    if (Date.now() >= deadline) {
+      throw new RebuildRefusal(
+        `writes into the old table begun before the cut-over at ${at} are still running after ${Math.round(timeoutMs / 1000)}s: ${pending.join(", ")}. ` +
+          `Let them finish, or stop them, and run again; cutoverTimeout sets how long the backfill waits.`,
+      );
+    }
+    const line = pending.join(", ");
+    if (line !== logged) run.log(`-- waiting for writes begun before the cut-over at ${at}: ${line}`);
+    logged = line;
+    await sleep(pollMs, run.signal);
+    pollMs = Math.min(pollMs * 2, 2_000);
+  }
+}
+
+/** The table an `INSERT INTO` statement writes to, or undefined for anything else (`INSERT INTO FUNCTION` included). */
+export function insertTarget(query: string, currentDatabase: string): { database: string; name: string } | undefined {
+  const text = query.replace(/^(?:\s+|--[^\n]*\n|\/\*[\s\S]*?\*\/)*/, "");
+  const part = "(`(?:[^`\\\\]|\\\\.)+`|\"(?:[^\"\\\\]|\\\\.)+\"|[A-Za-z_][A-Za-z0-9_]*)";
+  const m = new RegExp(`^INSERT\\s+INTO\\s+(?:TABLE\\s+)?${part}(?:\\s*\\.\\s*${part})?`, "i").exec(text);
+  if (!m || /^FUNCTION$/i.test(m[1]!)) return undefined;
+  const unquote = (s: string) => (/^[`"]/.test(s) ? s.slice(1, -1).replace(/\\(.)/g, "$1") : s);
+  return m[2] !== undefined ? { database: unquote(m[1]!), name: unquote(m[2]) } : { database: currentDatabase, name: unquote(m[1]!) };
+}
+
+/**
+ * The probe against the server. On a cluster of shards it reads every
+ * server's queries and buffers, and in a Replicated database every
+ * replica's, since a write can be running on any of them. A server that
+ * cannot be reached is left out rather than failing the wait.
+ */
+export function serverCutoverProbe(run: RebuildRun, o: Pick<RebuildObservation, "names" | "replicated">): CutoverProbe {
+  const { database, name } = o.names;
+  const cluster = run.sharding?.cluster ?? (o.replicated ? database : undefined);
+  const from = (table: string) => (cluster ? `clusterAllReplicas(${sqlString(cluster)}, system.${table})` : `system.${table}`);
+  // A server that cannot be reached runs no write the backfill could miss
+  // from here: its parts reach this one by replication, which the backfill
+  // waits for (`syncReplica`), or the verification finds them missing. So an
+  // unreachable replica or shard is skipped, as every other step goes on
+  // with the live ones; `clusterAllReplicas` counts each replica as a shard.
+  const opts = { ...(cluster ? { settings: { skip_unavailable_shards: "1" } } : {}), ...(run.signal ? { signal: run.signal } : {}) };
+  return {
+    now: () => serverNow(run.target),
+    async pendingWrites(cutover) {
+      const running = await clickhouseQuery<{ query_id: string; query: string; current_database: string; elapsed: number; now: string | number }>(
+        run.target.endpoint,
+        `SELECT query_id, query, current_database, elapsed, toUnixTimestamp64Milli(now64(3)) AS now FROM ${from("processes")} WHERE query_kind = 'Insert'`,
+        opts,
+      );
+      const queued = await clickhouseQuery<{ ids: string[] }>(
+        run.target.endpoint,
+        `SELECT entries.query_id AS ids FROM ${from("asynchronous_inserts")} ` +
+          `WHERE database = ${sqlString(database)} AND table = ${sqlString(name)} AND toUnixTimestamp64Milli(first_update) < ${cutover}`,
+        opts,
+      );
+      const ids = running
+        .filter((r) => {
+          const t = insertTarget(r.query, r.current_database);
+          return t?.database === database && t.name === name && Number(r.now) - Number(r.elapsed) * 1000 < cutover;
+        })
+        .map((r) => r.query_id);
+      for (const r of queued) ids.push(...r.ids);
+      return [...new Set(ids)];
+    },
+  };
+}
+
+export function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve, reject) => {
+    signal?.throwIfAborted();
+    const t = setTimeout(resolve, ms);
+    signal?.addEventListener(
+      "abort",
+      () => {
+        clearTimeout(t);
+        reject(signal.reason);
+      },
+      { once: true },
+    );
+  });
+}
+
 // ── swap ───────────────────────────────────────────────────────────────
 
 export interface SwapResult {

package/src/clickhouse/testing/fake-server.ts

@@ -52,6 +52,8 @@ export async function fakeClickHouse(objects: FakeObject[], options: FakeOptions
           .join(""),
       );
     }
+    // No SQL user-defined functions.
+    if (/FROM system\.functions/.test(body)) return ok("");
     const show = /^SHOW CREATE (DATABASE|TABLE) (`(?:[^`]|``)*`)(?:\.(`(?:[^`]|``)*`))?$/.exec(body.trim());
     if (show) {
       const [db, name] = show[3] ? [unquote(show[2]!), unquote(show[3])] : [undefined, unquote(show[2]!)];

package/src/clickhouse/testing/writable-server.ts

@@ -79,6 +79,8 @@ export async function writableClickHouse(initial: StoredObject[] = []): Promise<
           .join(""),
       };
     }
+    // No SQL user-defined functions.
+    if (/FROM system\.functions/.test(sql)) return { status: 200, text: "" };
     if (/FROM system\.mutations/.test(sql)) {
       const m = /database = '([^']*)' AND table = '([^']*)'/.exec(sql);
       return {

package/src/clickhouse/topology.ts

@@ -331,9 +331,10 @@ function renderCreate(tokens: Token[], node: CreateNode, topology: Topology): st
     return applyEdits(tokens, edits);
   }
 
-  const cluster = createClusterEdit(tokens, node, objectCluster(topology));
+  // A function belongs to no database, so in the replicated topology it reaches every replica only through the cluster.
+  const cluster = createClusterEdit(tokens, node, node.statement === "function" ? databaseCluster(topology, true) : objectCluster(topology));
   if (cluster) edits.push(cluster);
-  if (node.engine) {
+  if ((node.statement === "table" || node.statement === "view") && node.engine) {
     const declared: EngineClause = { name: node.engine.name, ...(node.engine.args ? { args: node.engine.args.map((a) => text(tokens, a)) } : {}) };
     const rendered = renderEngine(declared, topology);
     if (engineText(rendered) !== engineText(declared)) {
@@ -350,6 +351,13 @@ function renderOtherCreate(tokens: Token[], topology: Topology): string {
   w.accept("CREATE");
   if (w.accept("OR")) w.accept("REPLACE");
   w.accept("TEMPORARY");
+  // Access control (`./access.ts`) belongs to no database: like a database's own DDL, it reaches every replica through the cluster.
+  if (w.accept("USER", "ROLE", "POLICY", "QUOTA") || w.acceptSeq("ROW", "POLICY") || w.acceptSeq("SETTINGS", "PROFILE")) {
+    w.acceptSeq("IF", "NOT", "EXISTS") || w.acceptSeq("OR", "REPLACE");
+    if (!w.name()) return applyEdits(tokens, []);
+    const edit = onClusterEdit(w, databaseCluster(topology, true), w.end());
+    return applyEdits(tokens, edit ? [edit] : []);
+  }
   let database = false;
   if (w.accept("DATABASE")) database = true;
   else if (w.accept("MATERIALIZED", "LIVE", "WINDOW")) w.accept("VIEW");
@@ -421,10 +429,17 @@ export function renderStatement(sql: string, topology: Topology): string {
     w.p++;
     if (word(head, "DROP", "DETACH", "ATTACH")) w.accept("TEMPORARY");
     if (w.accept("DATABASE")) return renderAfterName(tokens, w, databaseCluster(topology, word(head, "DROP")));
+    // A function and access control belong to no database: like the database's own DDL, they reach every replica through the cluster.
+    if (w.accept("FUNCTION", "USER", "ROLE", "POLICY") || w.acceptSeq("ROW", "POLICY")) return renderAfterName(tokens, w, databaseCluster(topology, true));
     // `TRUNCATE t` may leave out TABLE.
     if (w.accept("TABLE", "VIEW", "DICTIONARY") || word(head, "TRUNCATE")) return renderAfterName(tokens, w, objectCluster(topology));
     return sql;
   }
+  if (word(head, "GRANT", "REVOKE")) {
+    w.p++;
+    const edit = onClusterEdit(w, databaseCluster(topology, true), w.end());
+    return applyEdits(tokens, edit ? [edit] : []);
+  }
   if (word(head, "RENAME")) {
     w.p++;
     if (w.accept("DATABASE")) return renderAtEnd(tokens, w, databaseCluster(topology));

package/src/clickhouse.ts

@@ -12,6 +12,12 @@ export {
   database,
   table,
   view,
+  dictionary,
+  func,
+  user,
+  role,
+  policy,
+  grant,
   literal,
   SqlLiteral,
   SqlTemplateError,
@@ -22,6 +28,19 @@ export {
   quoteIdentifier,
   type ClickHouseEntityType,
   type ClickHouseDatabase,
+  type ClickHouseDictionary,
+  type ClickHouseFunction,
+  type FunctionProps,
+  type ClickHouseUser,
+  type ClickHouseRole,
+  type ClickHouseRowPolicy,
+  type ClickHouseGrant,
+  type UserProps,
+  type RoleProps,
+  type RowPolicyProps,
+  type GrantProps,
+  type DictionaryAttributeDef,
+  type DictionaryProps,
   type ClickHouseRelation,
   type ClickHouseTable,
   type ClickHouseView,

package/src/codegen/classifier-page.ts

@@ -47,6 +47,8 @@ chant sql plan prod schema.json        # a build output against the prod server
 
 Both exit 2 when a change needs a rebuild. \`chant sql diff\` compares a pull request's base and head builds with no server. \`chant sql plan\` reads the server \`sql.profiles.<env>\` binds, and asks that server's formatter about any expression the normalization rules leave different, so the server's own rewriting (\`INTERVAL 1 DAY\` as \`toIntervalDay(1)\`, \`a+b*2\` as \`a + (b * 2)\`) is not reported as a change.
 
+\`chant sql plan\` also exits 2 when a declared database holds an object whose definition it cannot read. It names each one under \`Refused\` (\`unreadable\` in \`--json\`) instead of leaving it out. Dictionaries are read and planned like the other objects: any change but a comment replaces one (SQLCH245).
+
 ## After the plan
 
 The plan only reports. [\`clickhouseApply\`](../applying/) makes the metadata-only and background-rewrite changes, and refuses a rebuild the same way the plan does, sending nothing for that object. A rebuild runs as \`ClickHouseRebuildOp\`, a gated migration Op that creates the new table, backfills it, verifies it and swaps it in: see [Rebuilding a Table](../rebuild/). Both commands print the Op declaration to start from for each refused table.

package/src/codegen/registry.ts

@@ -17,6 +17,12 @@ const KINDS: Record<string, string> = {
   Table: CLICKHOUSE_ENTITY_TYPES.table,
   View: CLICKHOUSE_ENTITY_TYPES.view,
   MaterializedView: CLICKHOUSE_ENTITY_TYPES.materializedView,
+  Dictionary: CLICKHOUSE_ENTITY_TYPES.dictionary,
+  Function: CLICKHOUSE_ENTITY_TYPES.function,
+  User: CLICKHOUSE_ENTITY_TYPES.user,
+  Role: CLICKHOUSE_ENTITY_TYPES.role,
+  RowPolicy: CLICKHOUSE_ENTITY_TYPES.rowPolicy,
+  Grant: CLICKHOUSE_ENTITY_TYPES.grant,
   PostgresSchema: POSTGRES_ENTITY_TYPES.schema,
   PostgresTable: POSTGRES_ENTITY_TYPES.table,
   PostgresIndex: POSTGRES_ENTITY_TYPES.index,
@@ -29,6 +35,10 @@ const KINDS: Record<string, string> = {
   PostgresFunction: POSTGRES_ENTITY_TYPES.function,
   PostgresProcedure: POSTGRES_ENTITY_TYPES.procedure,
   PostgresTrigger: POSTGRES_ENTITY_TYPES.trigger,
+  PostgresPolicy: POSTGRES_ENTITY_TYPES.policy,
+  PostgresRole: POSTGRES_ENTITY_TYPES.role,
+  PostgresGrant: POSTGRES_ENTITY_TYPES.grant,
+  PostgresDefaultPrivileges: POSTGRES_ENTITY_TYPES.defaultPrivileges,
 };
 
 export function buildRegistry(): string {

package/src/config.ts

@@ -20,7 +20,8 @@
  * A profile is keyed by chant environment, as `grafana.profiles.<env>` is:
  * the server `chant import --from <env>`, `chant lifecycle diff <env> --live`
  * and `plan` read. Credentials are named by their environment variable, never
- * written in the config. An environment with no profile falls back to
+ * written in the config, or minted as a short-lived token (`password:
+ * { token: "rds-iam" }`; `./token-source.ts`). An environment with no profile falls back to
  * `CLICKHOUSE_URL`, `CLICKHOUSE_USER` and `CLICKHOUSE_PASSWORD`, or for
  * Postgres `POSTGRES_URL`, `POSTGRES_USER` and `POSTGRES_PASSWORD`. A profile
  * whose `url` is `postgres://` or `postgresql://` is a Postgres server.
@@ -46,6 +47,7 @@ import { PLANNED_SQL_DIALECTS, SQL_DIALECTS } from "./dialects";
 import { POSTGRES_PROVIDERS } from "./postgres/providers/types";
 import { POSTGRES_MAJORS } from "./spec/postgres-pin";
 import { parseTopology } from "./clickhouse/topology";
+import { tokenSourceSchema } from "./token-source";
 
 const envRef = z.strictObject({ env: z.string() });
 
@@ -85,8 +87,15 @@ export const sqlProfileSchema = z.strictObject({
   url: z.string(),
   /** The user, named by its environment variable. `default` when omitted. */
   user: envRef.optional(),
-  /** The password, named by its environment variable. */
-  password: envRef.optional(),
+  /**
+   * The password, named by its environment variable, or a token source that
+   * mints a short-lived one when a connection needs it (#3685):
+   * `{ token: "rds-iam" }`, `{ token: "cloud-sql-iam" }`, `{ token: "entra" }`
+   * (Postgres), or `{ token: "command", command: ["./mint.sh"] }`, whose
+   * standard output is the password. A token is minted again before it
+   * expires. See `./token-source.ts`.
+   */
+  password: z.union([envRef, tokenSourceSchema]).optional(),
   /**
    * The databases this environment's schema lives in. Import reads these and
    * nothing else; when omitted, every database except the server's own
@@ -146,6 +155,16 @@ export const sqlProfileSchema = z.strictObject({
    * `CONCURRENTLY` index build), in milliseconds. 0, no limit, when omitted.
    */
   scanTimeoutMs: z.number().int().nonnegative().optional(),
+  /**
+   * Postgres: whether chant manages access in this environment (#3681): the
+   * declared row-level security policies and each table's row-level security,
+   * the declared roles (created when missing, never dropped), and the
+   * privileges on the declared objects and the default privileges, which a
+   * plan compares and an apply grants and revokes. Off when omitted: the
+   * access declarations are not planned or applied, and the server's access
+   * is not read, so a project that manages access elsewhere is not affected.
+   */
+  access: z.boolean().optional(),
 });
 
 const dialectName = z.enum(SQL_DIALECTS);

package/src/core/references.ts

@@ -45,14 +45,16 @@ export function lineageJson(edges: readonly LineageEdge[], names: Map<Declarable
 }
 
 /**
- * Export names in creation order: an object after everything it references.
+ * Export names in creation order: an object after everything it references,
+ * and after the names in `extra` (an object read from a `.sql` file that a
+ * template names as text, `../sql-files.ts`).
  * Ties break by export name, so the order is the same however the files were
  * discovered. A reference cycle is an error naming the cycle.
  */
-export function applyOrder(objects: Map<string, Referencing>, names: Map<Declarable, string>): string[] {
+export function applyOrder(objects: Map<string, Referencing>, names: Map<Declarable, string>, extra?: ReadonlyMap<string, readonly string[]>): string[] {
   const deps = new Map<string, string[]>();
   for (const [name, obj] of objects) {
-    const on = new Set<string>();
+    const on = new Set<string>(extra?.get(name)?.filter((d) => d !== name && objects.has(d)) ?? []);
     for (const ref of obj.dependsOn) {
       const parent = isAttrRefLike(ref) ? (ref.parent.deref() as Declarable | undefined) : (ref as Declarable);
       const dep = parent ? names.get(parent) : undefined;

package/src/core/statements.ts

@@ -48,6 +48,14 @@ export interface StatementStep extends StepObject {
   waitsForMutation?: boolean;
   /** The statement removes data that is not recoverable (a column or table drop). */
   destructive?: boolean;
+  /**
+   * Postgres: the statement can fail on the rows already there (a NULL under
+   * SET NOT NULL, a duplicate under a unique index, a row a check or foreign
+   * key refuses). `sql` returns one row with one column, `n`, the count of
+   * rows it would fail on; run it before the first statement and stop when
+   * it is not zero (#3686).
+   */
+  precheck?: { sql: string; detail: string };
 }
 
 /** A change no statement makes in place, made by a migration Op. */
@@ -118,7 +126,9 @@ export function renderStatements(doc: SchemaStatements): string {
   for (const s of doc.steps) {
     if (s.kind === "statement") {
       const notes = [s.transactional || doc.dialect !== "postgres" ? "" : "outside a transaction", s.waitsForMutation ? "waits for its mutation" : "", s.destructive ? "destructive" : ""].filter(Boolean);
-      out.push(`-- ${s.object} (${s.name}): ${s.rule} ${s.class}${notes.length ? `, ${notes.join(", ")}` : ""}`, `${s.sql};`, "");
+      out.push(`-- ${s.object} (${s.name}): ${s.rule} ${s.class}${notes.length ? `, ${notes.join(", ")}` : ""}`);
+      if (s.precheck) out.push(`-- pre-check, must return 0 (${s.precheck.detail}): ${s.precheck.sql}`);
+      out.push(`${s.sql};`, "");
     } else if (s.kind === "op") {
       out.push(`-- ${s.object} (${s.name}): ${s.changes.map((c) => c.rule).join(", ")} made by ${s.op}, not a statement:`, `--   ${s.declaration}`, "");
     } else {

package/src/files/build-root.ts

@@ -0,0 +1,62 @@
+/**
+ * The `.sql` files in a project's source directory, read into declarations
+ * at build time (the plugin's `buildRoots` hook): each file's objects join
+ * the tagged templates' as if an import had written them, with the
+ * references between the two as interpolations (`../sql-files.ts`).
+ *
+ * The files are found by the walk source discovery uses, so `dist`,
+ * git-ignored paths, child projects and the project's `exclude` globs are
+ * left out. A file whose first lines carry `-- chant-discovery-skip` (or the
+ * `/* chant-discovery-skip *\/` form) is not read, for SQL that is not schema:
+ * seed data, a migration, a query.
+ *
+ * The dialect is the declarations': a project whose templates are Postgres
+ * reads its files as Postgres. With no templates it is `sql.dialect`, and
+ * ClickHouse when that is unset, as for the rest of the lexicon.
+ */
+
+import { readFileSync } from "node:fs";
+import { relative, resolve } from "node:path";
+import type { Declarable } from "@intentius/chant/declarable";
+import type { BuildRootContext, BuildRootContribution } from "@intentius/chant/lexicon";
+import { findSourceFiles, hasDiscoveryMarkerSync } from "@intentius/chant/discovery/files";
+import { readHead } from "@intentius/chant/discovery/walk";
+import { SQL_DIALECTS, type SqlDialect } from "../dialects";
+import { isPostgresObject } from "../postgres/entities";
+import { isClickHouseObject } from "../clickhouse/entities";
+import { sqlFileEntities } from "../sql-files";
+
+/** `-- chant-discovery-skip` on a line of its own, near the top. */
+const SQL_SKIP_MARKER = /^[ \t]*--[ \t]*chant-discovery-skip(?![\w-])/m;
+
+/** Whether a file is a `.sql` file to read: the name, and no skip or generated marker. */
+export function isSqlSourceFile(name: string, full: string): boolean {
+  if (!name.endsWith(".sql")) return false;
+  if (hasDiscoveryMarkerSync(full)) return false;
+  return !SQL_SKIP_MARKER.test(readHead(full, 1024));
+}
+
+/** The dialect the project's `.sql` files are read as. */
+export function filesDialect(entities: ReadonlyMap<string, Declarable> | undefined, config: Record<string, unknown>): SqlDialect {
+  for (const e of entities?.values() ?? []) {
+    if (isPostgresObject(e)) return "postgres";
+    if (isClickHouseObject(e)) return "clickhouse";
+  }
+  const dialect = (config.sql as { dialect?: unknown } | undefined)?.dialect;
+  if (typeof dialect === "string" && (SQL_DIALECTS as readonly string[]).includes(dialect)) return dialect as SqlDialect;
+  if (Array.isArray(dialect)) {
+    if (dialect.length === 1 && (SQL_DIALECTS as readonly string[]).includes(String(dialect[0]))) return dialect[0] as SqlDialect;
+    throw new Error(`sql: the project's .sql files hold no declaration that says their dialect, and sql.dialect lists ${dialect.join(", ")}; set sql.dialect to one, or declare one object with a tagged template`);
+  }
+  return "clickhouse";
+}
+
+/** The `buildRoots` contribution: every `.sql` file's declarations, or nothing when the source directory holds none. */
+export async function sqlFilesBuildRoot(ctx: BuildRootContext): Promise<BuildRootContribution> {
+  const sourceDir = ctx.sourceDir ?? resolve(ctx.projectRoot, typeof ctx.config.sourceDir === "string" ? ctx.config.sourceDir : ".");
+  const files = (await findSourceFiles(sourceDir, isSqlSourceFile)).sort();
+  if (files.length === 0) return { entities: new Map() };
+  const dialect = filesDialect(ctx.entities, ctx.config);
+  const sources = files.map((f) => ({ origin: relative(ctx.projectRoot, f) || f, ddl: readFileSync(f, "utf8") }));
+  return { entities: sqlFileEntities(dialect, sources, { known: ctx.entities ?? new Map() }) };
+}

package/src/files/clickhouse.ts

@@ -0,0 +1,52 @@
+/**
+ * A file of ClickHouse DDL as the objects it declares: one per `CREATE
+ * DATABASE`, `TABLE`, `VIEW`, `MATERIALIZED VIEW`, `DICTIONARY` or `FUNCTION`, as `chant import
+ * <file>.sql` reads them. Any other statement, and any statement the parser
+ * refuses, is a problem naming it.
+ *
+ * With `schema` (a database), an object's unqualified name is qualified with
+ * it; a function belongs to no database and keeps its name.
+ */
+
+import type { Declarable } from "@intentius/chant/declarable";
+import { isTrivia, tokenizeText } from "../clickhouse/tokens";
+import { parseCreate } from "../clickhouse/parser";
+import { database, dictionary, func, table, view } from "../clickhouse/entities";
+import { CLICKHOUSE_TAG_OF } from "../clickhouse/import/generator";
+import { describeStatement, type ImportedObject } from "../clickhouse/import/ir";
+import { firstLine, quoteBare, splitTokens, tokensText, type ReadOptions } from "./common";
+
+type Tag = (strings: TemplateStringsArray, ...values: unknown[]) => Declarable;
+const TAGS: Record<string, Tag> = { database, table, view, dictionary, func };
+
+/** The tag that declares an entity type. */
+export function clickhouseTag(type: string): Tag | undefined {
+  const name = CLICKHOUSE_TAG_OF[type];
+  return name ? TAGS[name] : undefined;
+}
+
+/** The objects a file of ClickHouse DDL declares. Problems are collected, not thrown, so a file is reported whole. */
+export function readClickHouse(ddl: string, options: ReadOptions, problems: string[]): ImportedObject[] {
+  const objects: ImportedObject[] = [];
+  for (const stmt of splitTokens(tokenizeText(ddl, 0), isTrivia)) {
+    let sql = tokensText(stmt);
+    if (!/^CREATE\b/i.test(sql)) {
+      problems.push(`not a CREATE statement: ${firstLine(sql)}`);
+      continue;
+    }
+    try {
+      const tokens = tokenizeText(sql, 0);
+      const node = parseCreate(tokens);
+      const name = tokens.slice(node.name.from, node.name.to).filter((t) => !isTrivia(t));
+      const [only] = name;
+      if (options.schema && node.statement !== "database" && node.statement !== "function" && only && name.length === 1) {
+        const at = tokens.indexOf(only);
+        sql = tokensText(tokens.slice(0, at)) + `${quoteBare(options.schema)}.` + tokensText(tokens.slice(at));
+      }
+      objects.push(describeStatement(sql));
+    } catch (e) {
+      problems.push(`does not parse (${e instanceof Error ? e.message : String(e)}): ${firstLine(sql)}`);
+    }
+  }
+  return objects;
+}

package/src/files/common.ts

@@ -0,0 +1,54 @@
+/** What reading a file of DDL shares across dialects. */
+
+import type { Token } from "../core/tokens";
+
+export interface ReadOptions {
+  /** Names the DDL in errors: a file's path, a command. */
+  origin: string;
+  /**
+   * The Postgres schema (ClickHouse database) for the names the DDL leaves
+   * unqualified, as an ORM does when the schema comes from its connection.
+   */
+  schema?: string;
+}
+
+/** A file of DDL with statements that declare nothing chant can hold: each problem names its statement. */
+export class SqlFileError extends Error {
+  constructor(
+    readonly origin: string,
+    readonly problems: readonly string[],
+  ) {
+    super(
+      `${origin}: ${problems.length === 1 ? "a statement" : `${String(problems.length)} statements`} chant cannot read as declarations:\n${problems.map((p) => `  - ${p}`).join("\n")}`,
+    );
+    this.name = "SqlFileError";
+  }
+}
+
+/** The statements of a file, each as its tokens (trivia kept inside, trimmed at the ends), split on top-level `;`. */
+export function splitTokens(tokens: readonly Token[], isTrivia: (t: Token) => boolean): Token[][] {
+  const out: Token[][] = [];
+  let current: Token[] = [];
+  const flush = () => {
+    let from = 0;
+    let to = current.length;
+    while (from < to && isTrivia(current[from]!)) from++;
+    while (to > from && isTrivia(current[to - 1]!)) to--;
+    if (to > from) out.push(current.slice(from, to));
+    current = [];
+  };
+  for (const t of tokens) {
+    if (t.kind === "punct" && t.text === ";") flush();
+    else current.push(t);
+  }
+  flush();
+  return out;
+}
+
+export const tokensText = (tokens: readonly Token[]): string => tokens.map((t) => t.text).join("");
+
+/** A statement's first line, for a message. */
+export const firstLine = (s: string): string => (s.split("\n")[0] ?? "").slice(0, 100);
+
+/** A name as SQL text: bare when it is a plain lower-case identifier, else double-quoted. */
+export const quoteBare = (name: string): string => (/^[a-z_][a-z0-9_]*$/.test(name) ? name : `"${name.replace(/"/g, '""')}"`);