@intentius/chant-lexicon-sql 0.117.0 → 0.118.0

package/src/postgres/import/generator.ts

@@ -24,16 +24,24 @@
 import type { GeneratedFile, TypeScriptGenerator } from "@intentius/chant/import/generator";
 import type { TemplateIR } from "@intentius/chant/import/parser";
 import { isTrivia, tokenizeText, type Token } from "../tokens";
-import { identValue } from "../parser";
+import { identValue, parseStatements, type NameNode, type StatementNode } from "../parser";
 import { POSTGRES_ENTITY_TYPES } from "../entity-types";
 
-interface Item {
+/** One object a template is written for, or (with no `ddl` read) one a template may reference. */
+export interface PgTemplateItem {
   exportName: string;
   type: string;
   schema?: string;
   name: string;
   ddl: string;
 }
+type Item = PgTemplateItem;
+
+/** A template's text: `parts` (template-literal safe) around one interpolation per entry of `refs`, an export name. */
+export interface TemplateBody {
+  parts: string[];
+  refs: string[];
+}
 
 const TAG: Record<string, string> = {
   [POSTGRES_ENTITY_TYPES.schema]: "schema",
@@ -54,37 +62,79 @@ const TAG: Record<string, string> = {
   [POSTGRES_ENTITY_TYPES.defaultPrivileges]: "grant",
 };
 
+/** The tag each entity type is declared with (`table`, `func`, ...). */
+export const POSTGRES_TAG_OF: Readonly<Record<string, string>> = TAG;
+
 /** Text that sits inside a template literal: a backquote and `${` escaped. */
 const templateSafe = (s: string) => s.replace(/\\(?=`|\$\{)/g, "\\\\").replace(/`/g, "\\`").replace(/\$\{/g, "\\${");
 
 const isName = (t: Token | undefined) => t !== undefined && (t.kind === "ident" || t.kind === "qident");
 
-/** `'app.invoice_seq'` as the qualified name it holds. */
+/** `'app.invoice_seq'` as the qualified name it holds, or the bare name when unqualified. */
 function regclassName(t: Token): string | undefined {
   if (t.kind !== "string" || !/^'.*'$/s.test(t.text)) return undefined;
   const inner = t.text.slice(1, -1).replace(/''/g, "'");
   const pieces = inner.match(/"(?:[^"]|"")*"|[^.]+/g)?.map((p) => identValue(p)) ?? [];
-  return pieces.length === 2 ? `${pieces[0]}.${pieces[1]}` : pieces.length === 1 ? `public.${pieces[0]}` : undefined;
+  return pieces.length === 2 ? `${pieces[0]}.${pieces[1]}` : pieces.length === 1 ? pieces[0] : undefined;
 }
 
-function templateBody(item: Item, byQualified: Map<string, Item>, schemaExport: Map<string, string>): { body: string; refs: string[] } {
+/** Words after which a bare name is a relation: `FROM t`, `JOIN t`, `REFERENCES t`. */
+const RELATION_WORDS = new Set(["FROM", "JOIN", "REFERENCES"]);
+
+/**
+ * The tokens where a bare (unqualified) name is another object's: a column's
+ * or a domain's type, the table of an index, trigger or policy, a trigger's
+ * function, a grant's objects. A `FROM`, `JOIN` or `REFERENCES` before a
+ * name is read off the tokens directly.
+ */
+function bareSpots(tokens: Token[]): Set<number> {
+  const out = new Set<number>();
+  let nodes: StatementNode[];
+  try {
+    nodes = parseStatements(tokens);
+  } catch {
+    return out;
+  }
+  const first = (from: number, to: number) => {
+    for (let i = from; i < to; i++) if (!isTrivia(tokens[i]!)) return out.add(i);
+    return out;
+  };
+  const name = (n: NameNode | undefined) => n && n.span.to - n.span.from > 0 && first(n.span.from, n.span.to);
+  for (const node of nodes) {
+    if (node.statement === "table") for (const c of node.columns) if (c.type) first(c.type.from, c.type.to);
+    if (node.statement === "domain") first(node.type.from, node.type.to);
+    if (node.statement === "index" || node.statement === "policy") name(node.table);
+    if (node.statement === "trigger") {
+      name(node.table);
+      name(node.function);
+    }
+    if (node.statement === "grant") for (const o of node.objects) name(o.name);
+  }
+  return out;
+}
+
+function templateBody(item: Item, byQualified: Map<string, Item>, schemaExport: Map<string, string>): TemplateBody {
   const tokens = tokenizeText(item.ddl, 0);
   const sig: number[] = [];
   tokens.forEach((t, i) => {
     if (!isTrivia(t)) sig.push(i);
   });
-  const replace = new Map<number, { to: number; text: string; ref: string }>();
+  const spots = bareSpots(tokens);
+  const replace = new Map<number, { to: number; ref: string }>();
+  const isSelf = (target: Item) => target === item || (target.exportName === item.exportName && target.type === item.type);
   for (let k = 0; k < sig.length; k++) {
     const a = tokens[sig[k]!]!;
     const prev = tokens[sig[k - 1] ?? -1];
     if (prev?.kind === "punct" && prev.text === ".") continue;
     // A regclass literal: nextval('app.seq'::regclass) or 'app.t'::regclass.
     if (a.kind === "string" && tokens[sig[k + 1] ?? -1]?.text === "::" && tokens[sig[k + 2] ?? -1]?.text.toLowerCase() === "regclass") {
-      const target = byQualified.get(regclassName(a) ?? "");
-      if (target && target !== item) {
+      const held = regclassName(a) ?? "";
+      // An unqualified name is the object in `public`, as the server's own printing has it, else the unqualified object of that name.
+      const target = (!held.includes(".") ? byQualified.get(`public.${held}`) : undefined) ?? byQualified.get(held);
+      if (target && !isSelf(target)) {
         const fn = tokens[sig[k - 2] ?? -1];
         const inCall = prev?.text === "(" && fn?.kind === "ident" && /^(nextval|currval|setval)$/i.test(fn.text);
-        replace.set(sig[k]!, { to: inCall ? sig[k + 2]! + 1 : sig[k]! + 1, text: `\${${target.exportName}}`, ref: target.exportName });
+        replace.set(sig[k]!, { to: inCall ? sig[k + 2]! + 1 : sig[k]! + 1, ref: target.exportName });
         k += inCall ? 2 : 0;
         continue;
       }
@@ -96,36 +146,85 @@ function templateBody(item: Item, byQualified: Map<string, Item>, schemaExport:
       const bySchemaWord = item.type === POSTGRES_ENTITY_TYPES.extension || item.type === POSTGRES_ENTITY_TYPES.grant || item.type === POSTGRES_ENTITY_TYPES.defaultPrivileges;
       if (bySchemaWord && isName(a) && prev?.kind === "ident" && prev.text.toUpperCase() === "SCHEMA") {
         const s = schemaExport.get(identValue(a));
-        if (s) replace.set(sig[k]!, { to: sig[k]! + 1, text: `\${${s}}`, ref: s });
+        if (s) replace.set(sig[k]!, { to: sig[k]! + 1, ref: s });
+        continue;
       }
+      // A bare name where an object is named: `REFERENCES users`, `ON users`, a column of type `status`.
+      const bare = isName(a) && (spots.has(sig[k]!) || (prev?.kind === "ident" && RELATION_WORDS.has(prev.text.toUpperCase())));
+      const target = bare ? byQualified.get(identValue(a)) : undefined;
+      if (target && !isSelf(target)) replace.set(sig[k]!, { to: sig[k]! + 1, ref: target.exportName });
       continue;
     }
     const schema = identValue(a);
     const name = identValue(b);
     const self = schema === item.schema && name === item.name;
     const target = byQualified.get(`${schema}.${name}`);
-    if (!self && target && target !== item) {
-      replace.set(sig[k]!, { to: sig[k + 2]! + 1, text: `\${${target.exportName}}`, ref: target.exportName });
+    if (!self && target && !isSelf(target)) {
+      replace.set(sig[k]!, { to: sig[k + 2]! + 1, ref: target.exportName });
       k += 2;
     } else if (self && schemaExport.has(schema)) {
-      replace.set(sig[k]!, { to: sig[k]! + 1, text: `\${${schemaExport.get(schema)}}`, ref: schemaExport.get(schema)! });
+      replace.set(sig[k]!, { to: sig[k]! + 1, ref: schemaExport.get(schema)! });
       k += 2;
     }
   }
-  let body = "";
+  const parts: string[] = [""];
   const refs: string[] = [];
   for (let i = 0; i < tokens.length; ) {
     const r = replace.get(i);
     if (r) {
-      body += r.text;
       refs.push(r.ref);
+      parts.push("");
       i = r.to;
       continue;
     }
-    body += templateSafe(tokens[i]!.text);
+    parts[parts.length - 1] += templateSafe(tokens[i]!.text);
     i++;
   }
-  return { body, refs };
+  return { parts, refs };
+}
+
+/** A template body as the text between the backquotes. */
+export const bodyText = (body: TemplateBody): string => body.parts.map((p, i) => (i === 0 ? p : `\${${body.refs[i - 1]}}${p}`)).join("");
+
+/**
+ * The template of each object in `items`, with the references to `targets`
+ * (the items themselves, and any object declared elsewhere that they may
+ * name) interpolated, and the items in the order a file declares them: each
+ * after what it references.
+ *
+ * A name is a reference where it is qualified (`app.users`), or bare where an
+ * object is named (`REFERENCES users`, `FROM users`, an index's table, a
+ * column's type) and an unqualified object has that name.
+ */
+export function postgresTemplates(items: readonly Item[], targets: readonly Item[] = items): { order: Item[]; bodies: Map<string, TemplateBody> } {
+  const byQualified = new Map<string, Item>();
+  const schemaExport = new Map<string, string>();
+  for (const it of targets) {
+    if (it.type === POSTGRES_ENTITY_TYPES.schema) schemaExport.set(it.name, it.exportName);
+    // An index or a trigger is never referenced by name in another object's DDL.
+    else if (it.type !== POSTGRES_ENTITY_TYPES.index && it.type !== POSTGRES_ENTITY_TYPES.trigger && it.type !== POSTGRES_ENTITY_TYPES.policy && it.type !== POSTGRES_ENTITY_TYPES.grant && it.type !== POSTGRES_ENTITY_TYPES.defaultPrivileges && it.type !== POSTGRES_ENTITY_TYPES.role && it.type !== POSTGRES_ENTITY_TYPES.extension) {
+      byQualified.set(it.schema ? `${it.schema}.${it.name}` : it.name, it);
+    }
+  }
+
+  const bodies = new Map(items.map((it) => [it.exportName, templateBody(it, byQualified, schemaExport)]));
+  const order: Item[] = [];
+  const done = new Set<string>();
+  const visiting = new Set<string>();
+  const byExport = new Map(items.map((it) => [it.exportName, it]));
+  const visit = (it: Item) => {
+    if (done.has(it.exportName) || visiting.has(it.exportName)) return;
+    visiting.add(it.exportName);
+    for (const ref of bodies.get(it.exportName)!.refs) {
+      const dep = byExport.get(ref);
+      if (dep) visit(dep);
+    }
+    visiting.delete(it.exportName);
+    done.add(it.exportName);
+    order.push(it);
+  };
+  for (const it of items) visit(it);
+  return { order, bodies };
 }
 
 const indent = (body: string) =>
@@ -151,39 +250,13 @@ export class PostgresGenerator implements TypeScriptGenerator {
       }));
     if (items.length === 0) return [];
 
-    const byQualified = new Map<string, Item>();
-    const schemaExport = new Map<string, string>();
-    for (const it of items) {
-      if (it.type === POSTGRES_ENTITY_TYPES.schema) schemaExport.set(it.name, it.exportName);
-      // An index or a trigger is never referenced by name in another object's DDL.
-      else if (it.schema && it.type !== POSTGRES_ENTITY_TYPES.index && it.type !== POSTGRES_ENTITY_TYPES.trigger && it.type !== POSTGRES_ENTITY_TYPES.policy) {
-        byQualified.set(`${it.schema}.${it.name}`, it);
-      }
-    }
-
-    const bodies = new Map(items.map((it) => [it.exportName, templateBody(it, byQualified, schemaExport)]));
-    const order: Item[] = [];
-    const done = new Set<string>();
-    const visiting = new Set<string>();
-    const byExport = new Map(items.map((it) => [it.exportName, it]));
-    const visit = (it: Item) => {
-      if (done.has(it.exportName) || visiting.has(it.exportName)) return;
-      visiting.add(it.exportName);
-      for (const ref of bodies.get(it.exportName)!.refs) {
-        const dep = byExport.get(ref);
-        if (dep) visit(dep);
-      }
-      visiting.delete(it.exportName);
-      done.add(it.exportName);
-      order.push(it);
-    };
-    for (const it of items) visit(it);
+    const { order, bodies } = postgresTemplates(items);
 
     const tags = [...new Set(order.map((it) => TAG[it.type]!))].sort();
     const lines: string[] = [];
     if (this.header) lines.push(...this.header.split("\n").map((l) => `// ${l}`), "");
     lines.push(`import { ${tags.join(", ")} } from "@intentius/chant-lexicon-sql/postgres";`, "");
-    for (const it of order) lines.push(`export const ${it.exportName} = ${TAG[it.type]}\`\n  ${indent(bodies.get(it.exportName)!.body)}\`;`, "");
+    for (const it of order) lines.push(`export const ${it.exportName} = ${TAG[it.type]}\`\n  ${indent(bodyText(bodies.get(it.exportName)!))}\`;`, "");
     return [{ path: "schema.ts", content: `${lines.join("\n").trimEnd()}\n` }];
   }
 }

package/src/postgres/live/bind.ts

@@ -8,6 +8,7 @@ import type { ChantConfig } from "@intentius/chant/config";
 import type { UnobservedReason } from "@intentius/chant/observation";
 import { connectPostgres, PostgresQueryError, type PostgresClient, type PostgresEndpoint } from "./client";
 import type { PostgresProvider } from "../providers/types";
+import { credentialSource, TokenSourceError } from "../../token-source";
 
 export interface PostgresTarget {
   endpoint: PostgresEndpoint;
@@ -58,12 +59,21 @@ export function resolvePostgresTarget(input: {
   if (profile && isPostgresUrl(profile.url)) {
     const source = `sql.profiles.${input.environment}`;
     const endpoint: PostgresEndpoint = { url: profile.url };
-    for (const key of ["user", "password"] as const) {
-      const ref = profile[key];
-      if (!ref) continue;
-      const value = env[ref.env];
-      if (value === undefined) return { reason: "no-credentials", detail: `${source}.${key} names ${ref.env}, which is not set` };
-      endpoint[key] = value;
+    if (profile.user) {
+      const value = env[profile.user.env];
+      if (value === undefined) return { reason: "no-credentials", detail: `${source}.user names ${profile.user.env}, which is not set` };
+      endpoint.user = value;
+    }
+    const password = profile.password;
+    if (password && "env" in password) {
+      const value = env[password.env];
+      if (value === undefined) return { reason: "no-credentials", detail: `${source}.password names ${password.env}, which is not set` };
+      endpoint.password = value;
+    } else if (password) {
+      if (password.token === "rds-iam" && endpoint.user === undefined) {
+        return { reason: "no-credentials", detail: `${source}.password is an rds-iam token, which is minted for one user, and ${source}.user is not set` };
+      }
+      endpoint.token = credentialSource(password, { url: profile.url, ...(endpoint.user !== undefined ? { user: endpoint.user } : {}), env, source: `${source}.password` });
     }
     const timeouts = Object.fromEntries(
       (["lockTimeoutMs", "statementTimeoutMs", "scanTimeoutMs"] as const).filter((k) => profile[k] !== undefined).map((k) => [k, profile[k]]),
@@ -142,6 +152,7 @@ const CREDENTIAL_STATES = new Set(["28000", "28P01", "42501"]);
 /** What a failed bind or read means, in the observation vocabulary. */
 export function classifyPostgresFailure(err: unknown): { reason: UnobservedReason; detail: string } {
   if (err instanceof PostgresBindingError) return { reason: err.unresolved.reason, detail: err.unresolved.detail };
+  if (err instanceof TokenSourceError) return { reason: "no-credentials", detail: err.message };
   if (err instanceof PostgresQueryError && err.code && CREDENTIAL_STATES.has(err.code)) {
     return { reason: "no-credentials", detail: `${err.message} (the server refused the credentials, SQLSTATE ${err.code})` };
   }

package/src/postgres/live/client.ts

@@ -8,11 +8,15 @@
  * search path.
  */
 
+import type { CredentialSource } from "../../token-source";
+
 export interface PostgresEndpoint {
   /** A `postgres://` connection URL, without a password. */
   url: string;
   user?: string;
   password?: string;
+  /** Mints the password at each connect, in place of `password` (a profile's token source, #3685). */
+  token?: CredentialSource;
 }
 
 /** What chant reads and writes through: one connection. */
@@ -54,7 +58,8 @@ export async function connectPostgres(endpoint: PostgresEndpoint, options: { app
   // node-postgres lets the connection string's fields win over the config's, so the credentials go into the URL.
   const url = new URL(endpoint.url);
   if (endpoint.user !== undefined) url.username = encodeURIComponent(endpoint.user);
-  if (endpoint.password !== undefined) url.password = encodeURIComponent(endpoint.password);
+  const password = endpoint.token ? await endpoint.token.get() : endpoint.password;
+  if (password !== undefined) url.password = encodeURIComponent(password);
   const client = new pg.Client({ connectionString: url.toString(), application_name: options.applicationName ?? "chant" });
   // A server that goes away emits an error event; the next query fails with it instead of the process crashing.
   client.on("error", () => undefined);
@@ -63,7 +68,10 @@ export async function connectPostgres(endpoint: PostgresEndpoint, options: { app
     await client.query("SELECT pg_catalog.set_config('search_path', '', false)");
   } catch (err) {
     await client.end().catch(() => undefined);
-    throw wrap(err);
+    const wrapped = wrap(err);
+    // A refused token is not reused: the next connect mints another.
+    if (endpoint.token && (wrapped.code === "28P01" || wrapped.code === "28000")) endpoint.token.invalidate();
+    throw wrapped;
   }
   let ended = false;
   return {

package/src/postgres/plan/plan.test.ts

@@ -82,7 +82,7 @@ describe("columns", () => {
     expect(rules(diff([table(ORDERS)], [table("CREATE TABLE app.orders (id bigint PRIMARY KEY, amount numeric(12,2) DEFAULT 0, note varchar(20) NOT NULL, status text)")]))).toEqual([
       ["columns.amount.default", "SQLPG209", "metadata"],
       ["columns.amount.notNull", "SQLPG211", "metadata"],
-      ["columns.note.notNull", "SQLPG210", "rewrite"],
+      ["columns.note.notNull", "SQLPG210", "validate"],
     ]);
     const renamed = diff([table(ORDERS)], [table(`CREATE TABLE app.orders (id bigint PRIMARY KEY, amount numeric(12,2) NOT NULL, note varchar(20),
       state text -- previously: status
@@ -113,7 +113,7 @@ describe("constraints", () => {
     );
     expect(rules(d)).toEqual([
       ["constraints.unique", "SQLPG221", "concurrently"],
-      ["constraints.a", "SQLPG218", "rewrite"],
+      ["constraints.a", "SQLPG218", "validate"],
       ["constraints.b", "SQLPG217", "metadata"],
       ["constraints.c", "SQLPG219", "validate"],
       ["constraints.exclude", "SQLPG222", "rewrite"],

package/src/postgres/plan/rules.ts

@@ -109,9 +109,9 @@ export const PG_CLASSIFIER_RULES = {
   SQLPG209: rule("SQLPG209", "metadata", "Change a column's default", "SET DEFAULT and DROP DEFAULT change only the catalog; existing rows keep their values.", ALTER_TABLE),
   SQLPG210: rule(
     "SQLPG210",
-    "rewrite",
+    "validate",
     "Set NOT NULL",
-    "SET NOT NULL reads the whole table under ACCESS EXCLUSIVE, unless a valid CHECK (column IS NOT NULL) constraint already proves it (since 12). Declare that check NOT VALID, validate it, then set NOT NULL; 18 can also add NOT NULL ... NOT VALID.",
+    "SET NOT NULL alone reads the whole table under ACCESS EXCLUSIVE, unless a valid CHECK (column IS NOT NULL) constraint already proves it (since 12). So it is made as four statements: that check added NOT VALID, validated under SHARE UPDATE EXCLUSIVE while writes go on, SET NOT NULL, which the check proves without a scan, and the check dropped. Rows where the column is NULL fail the validation; the first statement's pre-check counts them.",
     ALTER_TABLE,
   ),
   SQLPG211: rule("SQLPG211", "metadata", "Drop NOT NULL", "DROP NOT NULL changes only the catalog.", ALTER_TABLE),
@@ -135,16 +135,16 @@ export const PG_CLASSIFIER_RULES = {
   ),
   SQLPG218: rule(
     "SQLPG218",
-    "rewrite",
+    "validate",
     "Add a CHECK constraint",
-    "ADD CONSTRAINT ... CHECK reads every row under ACCESS EXCLUSIVE. Declare it NOT VALID, then remove NOT VALID to validate it under SHARE UPDATE EXCLUSIVE (SQLPG220).",
+    "ADD CONSTRAINT ... CHECK alone reads every row under ACCESS EXCLUSIVE. On a table that exists it is made as two statements: the check added NOT VALID (SQLPG217), which reads no rows, then VALIDATE CONSTRAINT (SQLPG220) in a transaction of its own, which reads them under SHARE UPDATE EXCLUSIVE while writes go on. Rows the check refuses fail the validation; the first statement's pre-check counts them.",
     ALTER_TABLE,
   ),
   SQLPG219: rule(
     "SQLPG219",
     "validate",
     "Add a foreign key",
-    "ADD FOREIGN KEY reads every row under SHARE ROW EXCLUSIVE on both tables, which blocks writes to them for the scan. Declare it NOT VALID, then remove NOT VALID to validate it under SHARE UPDATE EXCLUSIVE (SQLPG220).",
+    "ADD FOREIGN KEY alone reads every row under SHARE ROW EXCLUSIVE on both tables, which blocks writes to them for the scan. On a table that exists it is made as two statements: the key added NOT VALID (SQLPG217), which takes that lock briefly and reads no rows, then VALIDATE CONSTRAINT (SQLPG220) in a transaction of its own, which reads them under SHARE UPDATE EXCLUSIVE while writes go on. Orphan rows fail the validation; the first statement's pre-check counts them.",
     ALTER_TABLE,
   ),
   SQLPG220: rule(

package/src/serializer.ts

@@ -12,7 +12,8 @@
  *
  * The order is the dependency order of the references: a view after the
  * tables it reads, a materialized view after its target, a table after its
- * database. Ties break by export name, so the output is the same however the
+ * database. An object read from a `.sql` file (`./sql-files.ts`) that a
+ * template names in its text counts as referenced too. Ties break by export name, so the output is the same however the
  * files were discovered. A reference cycle is an error naming the cycle.
  *
  * One build holds one dialect: the dialect is the document's, and a project
@@ -35,6 +36,7 @@ import { isClickHouseObject, type ClickHouseObject } from "./clickhouse/entities
 import { isPostgresObject, type PostgresObject } from "./postgres/entities";
 import { POSTGRES_LATEST_MAJOR } from "./spec/postgres-pin";
 import { applyOrder, lineageJson, referenceName, type LineageEdge } from "./core/references";
+import { sqlFileReferences } from "./sql-files";
 
 export { applyOrder } from "./core/references";
 
@@ -44,7 +46,7 @@ export const CLICKHOUSE_DDL_FILE = "clickhouse.sql";
 /** The Postgres statements' file, beside the primary output. */
 export const POSTGRES_DDL_FILE = "postgres.sql";
 
-function objectJson(name: string, obj: ClickHouseObject, names: Map<Declarable, string>): Record<string, unknown> {
+function objectJson(name: string, obj: ClickHouseObject, names: Map<Declarable, string>, extra: readonly string[] = []): Record<string, unknown> {
   const props = { ...(obj.props as Record<string, unknown>) };
   delete props.source;
   const ddl = props.ddl;
@@ -54,7 +56,7 @@ function objectJson(name: string, obj: ClickHouseObject, names: Map<Declarable,
   if (props.lineage) {
     props.lineage = lineageJson(props.lineage as LineageEdge[], names);
   }
-  const dependsOn = [...new Set(obj.dependsOn.map((r) => referenceName(r, names)).filter((n) => n !== undefined))];
+  const dependsOn = [...new Set([...obj.dependsOn.map((r) => referenceName(r, names)).filter((n) => n !== undefined), ...extra])];
   return JSON.parse(
     JSON.stringify({ export: name, type: obj.entityType, sqlName: obj.sqlName, ...props, dependsOn, ddl }),
   ) as Record<string, unknown>;
@@ -68,12 +70,12 @@ function postgresJsonValue(v: unknown, names: Map<Declarable, string>): unknown
   return v;
 }
 
-function postgresObjectJson(name: string, obj: PostgresObject, names: Map<Declarable, string>): Record<string, unknown> {
+function postgresObjectJson(name: string, obj: PostgresObject, names: Map<Declarable, string>, extra: readonly string[] = []): Record<string, unknown> {
   const props = { ...(obj.props as Record<string, unknown>) };
   delete props.source;
   const ddl = props.ddl;
   delete props.ddl;
-  const dependsOn = [...new Set(obj.dependsOn.map((r) => referenceName(r, names)).filter((n) => n !== undefined))];
+  const dependsOn = [...new Set([...obj.dependsOn.map((r) => referenceName(r, names)).filter((n) => n !== undefined), ...extra])];
   return JSON.parse(
     JSON.stringify({ export: name, type: obj.entityType, sqlName: obj.sqlName, ...(postgresJsonValue(props, names) as object), dependsOn, ddl }),
   ) as Record<string, unknown>;
@@ -92,12 +94,14 @@ function serializePostgres(
 ): SerializerResult {
   const names = new Map<Declarable, string>();
   for (const [name, entity] of entities) names.set(entity, name);
-  const order = applyOrder(objects, names);
+  // A template names an object read from a `.sql` file as text: that is an edge too.
+  const extra = sqlFileReferences("postgres", objects);
+  const order = applyOrder(objects, names, extra);
   const doc = {
     dialect: "postgres",
     postgresMajor: targetMajor(config),
     applyOrder: order,
-    objects: order.map((n) => postgresObjectJson(n, objects.get(n)!, names)),
+    objects: order.map((n) => postgresObjectJson(n, objects.get(n)!, names, extra.get(n))),
   };
   const ddl = order.map((n) => `${(objects.get(n)!.props as { ddl: string }).ddl};`).join("\n\n");
   return {
@@ -130,11 +134,12 @@ export const sqlSerializer: Serializer = {
     const names = new Map<Declarable, string>();
     for (const [name, entity] of entities) names.set(entity, name);
 
-    const order = applyOrder(objects, names);
+    const extra = sqlFileReferences("clickhouse", objects);
+    const order = applyOrder(objects, names, extra);
     const doc = {
       dialect: "clickhouse",
       applyOrder: order,
-      objects: order.map((n) => objectJson(n, objects.get(n)!, names)),
+      objects: order.map((n) => objectJson(n, objects.get(n)!, names, extra.get(n))),
     };
     const ddl = order
       .map((n) => `${(objects.get(n)!.props as { ddl: string }).ddl};`)

package/src/skills/chant-sql-postgres-plan.md

@@ -83,9 +83,9 @@ Every rule has an id in the SQLPG2xx range, names its class and cites the Postgr
 | column type to a binary-coercible type (longer varchar, text, wider numeric) | SQLPG206 | metadata | indexes on the column may still be rebuilt |
 | column type with a rewrite (integer to bigint, a shorter varchar) | SQLPG207 | rewrite | |
 | column type across kinds (text to integer) | SQLPG208 | expand | needs USING and breaks readers |
-| set NOT NULL | SQLPG210 | rewrite | a valid `CHECK (col IS NOT NULL)` already in place makes it metadata |
-| add a CHECK | SQLPG218 | rewrite | declare it `NOT VALID`, then validate |
-| add a foreign key | SQLPG219 | validate | `SHARE ROW EXCLUSIVE` on both tables for the scan; `NOT VALID` first avoids it |
+| set NOT NULL | SQLPG210 | validate | made as a `NOT VALID` check, `VALIDATE`, `SET NOT NULL`, the check dropped; a valid `CHECK (col IS NOT NULL)` already in place makes it metadata |
+| add a CHECK | SQLPG218 | validate | made as `ADD CONSTRAINT ... NOT VALID`, then `VALIDATE` in its own transaction; writes go on during the scan |
+| add a foreign key | SQLPG219 | validate | made as `ADD CONSTRAINT ... NOT VALID`, then `VALIDATE` in its own transaction; writes go on during the scan |
 | add a constraint `NOT VALID` | SQLPG217 | metadata | checks new rows only |
 | validate a constraint | SQLPG220 | validate | `SHARE UPDATE EXCLUSIVE`, so reads and writes go on |
 | add a primary key or unique constraint | SQLPG221 | concurrently | build the unique index CONCURRENTLY, then `ADD CONSTRAINT ... USING INDEX` |

package/src/skills/chant-sql-rebuild.md

@@ -26,7 +26,7 @@ export const { op } = ClickHouseRebuildOp({
 });
 ```
 
-Options: `name`, `env`, `table`, `dualWrite` (required); `output` (default `dist/schema.json`), `path`, `build` (default true runs `chant build` first), `retain` (default `7d`), `gate` (the swap gate: `gate`, `timeout`, `description`, `approval`), `dropGate`, `writesGate` (app mode), `backfillTimeout` (default `6h`), `mutationTimeout` (default `10m`), `replicaTimeout` (default `2m`, Replicated databases only), `stack`, `ownershipEnv`.
+Options: `name`, `env`, `table`, `dualWrite` (required); `output` (default `dist/schema.json`), `path`, `build` (default true runs `chant build` first), `retain` (default `7d`), `gate` (the swap gate: `gate`, `timeout`, `description`, `approval`), `dropGate`, `writesGate` (app mode), `backfillTimeout` (default `6h`), `mutationTimeout` (default `10m`), `replicaTimeout` (default `2m`, Replicated databases only), `cutoverTimeout` (default `10m`, materialized-view mode only), `stack`, `ownershipEnv`.
 
 On a `cluster:<name>` topology with more than one shard, the backfill and the verification work per shard through the profile's server, with a receipt per shard and partition; each shard needs its own `{shard}` macro.
 
@@ -34,7 +34,7 @@ Run as one step of a change approved elsewhere: `gates: "outer"` leaves out the
 
 Pick the dual-write mode:
 
-- `{ mode: "materialized-view", cutoverColumn: "ts", cutoverDelay?: "1m" }` when the table has a time column rows arrive in order of. Writes keep flowing: a materialized view sends rows at or after the cut-over to the new table, the backfill copies the rows before it.
+- `{ mode: "materialized-view", cutoverColumn: "ts", cutoverDelay?: "5s" }` when the table has a time column rows arrive in order of. Writes keep flowing: a materialized view sends rows at or after the cut-over to the new table, the backfill copies the rows before it. The backfill starts once the server's clock has passed the cut-over and every INSERT into the old table begun before it has finished, asynchronous inserts still in their buffer included; `cutoverTimeout` bounds the second wait. Set `cutoverDelay` longer than the batches of a writer that holds rows before inserting them.
 - `{ mode: "app" }` otherwise. A gate, `<name>-writes-stopped`, waits for someone to confirm the application stopped writing to the table; writes resume after the swap.
 
 ## Run it
@@ -57,6 +57,7 @@ The swap gate is bound to a digest of the plan and the verification (row counts
 - A run stopped during the backfill (Ctrl-C, a killed job) is not a failure. The next run resumes from the receipts in `chant_receipts.receipts` on the same server (in `<db>.__chant_receipts` on a Replicated database), and a partition whose copy was cut off before its receipt is cleared and copied again, so no row is copied twice.
 - A failed step (a failed copy after its retries, a verification mismatch, a refused declaration) runs onFailure: the new table `<t>__chant_new` and the dual-write view `<t>__chant_dual` are dropped, and the next run starts from a new table. With `onFailure: "keep"` they stay, and the next run resumes; drop them by hand to start over. The original table is untouched until the swap.
 - A verification mismatch in materialized-view mode usually means rows arrived with a time before the cut-over later than `cutoverDelay`. Raise the delay and run again.
+- A backfill that stops naming writes "begun before the cut-over" waited `cutoverTimeout` for INSERTs into the old table that were still running. Let them finish, or stop them, and run again.
 - The Op refuses a database that is neither Atomic nor Replicated (`EXCHANGE TABLES` needs one of them), a table whose change is not a rebuild (use the applier), and an object already under one of its working names that is not its own. In a Replicated database it also refuses a table, or a declaration, whose engine is not `Replicated*MergeTree`: a plain MergeTree keeps different rows on each replica and one replica's copy would lose the others'.
 
 ## On a Replicated database

package/src/skills/chant-sql.md

@@ -1,6 +1,6 @@
 ---
 skill: chant-sql
-description: Declare ClickHouse databases, tables, views and materialized views as SQL-shaped tagged templates, with references, lineage and the checks that run on them
+description: Declare ClickHouse databases, tables, views, materialized views, dictionaries and functions as SQL-shaped tagged templates, with references, lineage and the checks that run on them
 user-invocable: true
 ---
 # Declaring a ClickHouse schema with chant
@@ -33,6 +33,12 @@ export const byKind = view`
 
 The export name (`events`) is the object's identity in chant. The name in the SQL (`analytics.events`) is its name in the database. A materialized view is a `view` template holding `CREATE MATERIALIZED VIEW`.
 
+A dictionary is a `dictionary` template holding `CREATE DICTIONARY`, with its attributes, `PRIMARY KEY`, `SOURCE`, `LAYOUT` and `LIFETIME`. Its attributes are columns (`${ratesDict.columns.rate}`). Any change but its comment is SQLCH245, and the applier replaces it with `CREATE OR REPLACE DICTIONARY`. Keep passwords out of `SOURCE`: name a server-side named collection instead.
+
+A SQL user-defined function is a `func` template holding `CREATE FUNCTION name AS (x) -> expr`, with no database in its name. A changed expression is SQLCH260 (`CREATE OR REPLACE FUNCTION`). It carries no ownership marker, so chant never drops one.
+
+Users, roles, row policies and grants are `user`, `role`, `policy` and `grant` templates. Never write a password: `IDENTIFIED BY` is refused, a user declared without `IDENTIFIED` is created by the environment with its password, and chant manages the rest of it. Grants are compared per grantee: every `grant` naming a grantee adds up to its complete list, and anything else it holds is revoked (SQLCH274). None of these is ever dropped by chant.
+
 ## Interpolations
 
 | Value | Means | Renders as |