@intentius/chant-lexicon-sql 0.117.0 → 0.118.0

package/src/sql-files.test.ts

@@ -0,0 +1,188 @@
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { mkdir, realpath, rm, writeFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { build } from "@intentius/chant/build";
+import { readSqlFile, sqlFileDeclarations, SqlFileError } from "./sql-files";
+import { sqlPlugin } from "./plugin";
+import { sqlSerializer } from "./serializer";
+import { guessDialect, SqlFileParser } from "./import-parser";
+
+const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
+
+const PG_DDL = `BEGIN;
+CREATE TYPE app.status AS ENUM ('placed', 'paid');
+CREATE TABLE app.orders (
+  id bigint PRIMARY KEY,
+  user_id bigint NOT NULL,
+  status app.status NOT NULL
+);
+CREATE TABLE app.users (id bigint PRIMARY KEY, email text NOT NULL);
+ALTER TABLE app.orders ADD CONSTRAINT orders_user_fk FOREIGN KEY (user_id) REFERENCES app.users (id);
+CREATE INDEX orders_user_idx ON app.orders (user_id);
+COMMENT ON TABLE app.users IS 'People';
+COMMENT ON COLUMN app.users.email IS 'Login';
+ALTER TABLE app.users ENABLE ROW LEVEL SECURITY;
+COMMIT;
+`;
+
+describe("reading a file of Postgres DDL", () => {
+  test("each CREATE is an object, with its comments, row security and added constraints folded in", () => {
+    const objects = readSqlFile("postgres", PG_DDL, { origin: "schema.sql" });
+    expect(objects.map((o) => [o.export, o.type, o.name])).toEqual([
+      ["status", "Postgres::Enum", "app.status"],
+      ["orders", "Postgres::Table", "app.orders"],
+      ["users", "Postgres::Table", "app.users"],
+      ["ordersUserIdx", "Postgres::Index", "app.orders_user_idx"],
+    ]);
+    expect(objects[1]!.ddl).toContain("CONSTRAINT orders_user_fk FOREIGN KEY (user_id) REFERENCES app.users (id)\n)");
+    expect(objects[2]!.ddl).toBe(
+      "CREATE TABLE app.users (id bigint PRIMARY KEY, email text NOT NULL);\nCOMMENT ON TABLE app.users IS 'People';\nCOMMENT ON COLUMN app.users.email IS 'Login';\nALTER TABLE app.users ENABLE ROW LEVEL SECURITY",
+    );
+  });
+
+  test("a statement that declares nothing or does not parse is an error naming it, and every one is named", () => {
+    const read = () => readSqlFile("postgres", "CREATE TABLE a (id int);\nINSERT INTO a VALUES (1);\nCREATE TABLE b (id int,);\nCOMMENT ON TABLE c IS 'x';", { origin: "db/schema.sql" });
+    expect(read).toThrow(SqlFileError);
+    expect(read).toThrow(/^db\/schema\.sql: 3 statements chant cannot read as declarations/);
+    expect(read).toThrow(/not a statement that declares an object: INSERT INTO a VALUES \(1\)/);
+    expect(read).toThrow(/does not parse \(.*\): CREATE TABLE b \(id int,\)/);
+    expect(read).toThrow(/COMMENT ON TABLE for an object the file does not create: COMMENT ON TABLE c IS 'x'/);
+  });
+
+  test("schema qualifies the names the DDL leaves unqualified", () => {
+    const { content } = sqlFileDeclarations("postgres", "CREATE TYPE s AS ENUM ('a');\nCREATE TABLE u (id int PRIMARY KEY, k s);\nCREATE TABLE o (u int REFERENCES u (id));", { origin: "x", schema: "app" });
+    expect(content).toContain("CREATE TABLE app.u (id int PRIMARY KEY, k ${s})");
+    expect(content).toContain("CREATE TABLE app.o (u int REFERENCES ${u} (id))");
+  });
+});
+
+describe("chant import of a .sql file", () => {
+  test("the dialect is read off the statements, and Postgres DDL imports as Postgres declarations", () => {
+    expect(guessDialect(PG_DDL)).toBe("postgres");
+    expect(guessDialect("CREATE TABLE a.t (x UInt8) ENGINE = Log")).toBe("clickhouse");
+    expect(guessDialect("CREATE VIEW a.v AS SELECT 1")).toBe("clickhouse");
+    expect(guessDialect("CREATE FUNCTION f AS (x) -> x + 1")).toBe("clickhouse");
+    expect(guessDialect("CREATE FUNCTION app.f() RETURNS int LANGUAGE sql AS $$ SELECT 1 $$")).toBe("postgres");
+    expect(new SqlFileParser().parse(PG_DDL).resources.map((r) => r.type)).toEqual(["Postgres::Enum", "Postgres::Table", "Postgres::Table", "Postgres::Index"]);
+  });
+
+  test("a statement it cannot read stops the import, named", () => {
+    expect(() => new SqlFileParser("clickhouse").parse("CREATE DATABASE a;\nINSERT INTO a.t VALUES (1)")).toThrow(/not a CREATE statement: INSERT INTO a\.t VALUES \(1\)/);
+  });
+});
+
+/** A project directory under the repository, so its imports resolve. */
+async function project(name: string, files: Record<string, string>): Promise<string> {
+  const path = join(repoRoot, ".cache", `sql-3646-${name}-${process.pid}`);
+  await rm(path, { recursive: true, force: true });
+  await mkdir(join(path, "src"), { recursive: true });
+  const dir = await realpath(path);
+  for (const [file, content] of Object.entries(files)) await writeFile(join(dir, "src", file), content);
+  return dir;
+}
+
+/** A build document with each DDL's whitespace runs collapsed. */
+const squeeze = (doc: unknown): unknown => JSON.parse(JSON.stringify(doc, (k, v: unknown) => (k === "ddl" && typeof v === "string" ? v.replace(/\s+/g, " ") : v)));
+
+async function buildProject(dir: string, config: Record<string, unknown> = {}) {
+  const result = await build(join(dir, "src"), [sqlSerializer], undefined, {
+    lexicons: ["sql"],
+    intrinsics: sqlPlugin.intrinsics?.() ?? [],
+    buildRoots: [(ctx) => sqlPlugin.buildRoots!({ projectRoot: dir, config, entities: ctx.entities, ...(ctx.sourceDir ? { sourceDir: ctx.sourceDir } : {}) })],
+  });
+  const out = result.outputs.get("sql");
+  const primary = typeof out === "string" ? out : out?.primary;
+  return { errors: result.errors.map((e) => e.message), doc: primary ? (JSON.parse(primary) as { applyOrder: string[]; objects: Array<Record<string, unknown>> }) : undefined };
+}
+
+const CH_DDL = `CREATE DATABASE analytics;
+CREATE TABLE analytics.events (ts DateTime, kind String) ENGINE = MergeTree ORDER BY ts;
+CREATE TABLE analytics.counts (kind String, n UInt64) ENGINE = SummingMergeTree ORDER BY kind;
+CREATE MATERIALIZED VIEW analytics.counts_mv TO analytics.counts AS SELECT kind, count() AS n FROM analytics.events GROUP BY kind;
+CREATE DICTIONARY analytics.kinds (kind String, label String DEFAULT '') PRIMARY KEY kind SOURCE(clickhouse(table 'counts' db 'analytics')) LAYOUT(complex_key_hashed()) LIFETIME(300);
+CREATE FUNCTION kind_label AS (k) -> upper(k);
+`;
+
+describe("chant build reads the .sql files in the source directory", () => {
+  const dirs: string[] = [];
+  afterAll(async () => {
+    for (const d of dirs) await rm(d, { recursive: true, force: true });
+  });
+
+  for (const [dialect, ddl] of [
+    ["postgres", PG_DDL.replace("CREATE TYPE", "CREATE SCHEMA app;\nCREATE TYPE")],
+    ["clickhouse", CH_DDL],
+  ] as const) {
+    test(`${dialect}: the same build as the declarations chant import writes for the file`, async () => {
+      const fromSql = await project(`${dialect}-sql`, { "schema.sql": ddl });
+      const fromTs = await project(`${dialect}-ts`, { "schema.ts": sqlFileDeclarations(dialect, ddl, { origin: "schema.sql" }).content });
+      dirs.push(fromSql, fromTs);
+      const a = await buildProject(fromSql, { sql: { dialect } });
+      const b = await buildProject(fromTs, { sql: { dialect } });
+      expect(a.errors).toEqual([]);
+      expect(b.errors).toEqual([]);
+      // The import indents a template's lines under its declaration; the statements are otherwise the same.
+      expect(squeeze(a.doc)).toEqual(squeeze(b.doc));
+      expect(a.doc!.objects.length).toBeGreaterThan(2);
+    });
+  }
+
+  let mixed: string;
+  beforeAll(async () => {
+    mixed = await project("mixed", {
+      "app.ts": `import { schema, view } from "@intentius/chant-lexicon-sql/postgres";
+export const app = schema\`CREATE SCHEMA app\`;
+export const activeUsers = view\`CREATE VIEW app.active_users AS SELECT id FROM app.users\`;
+`,
+      "users.sql": "CREATE TABLE app.users (id bigint PRIMARY KEY);\nCREATE TABLE app.notes (user_id bigint REFERENCES app.users (id));\n",
+    });
+    dirs.push(mixed);
+  });
+
+  test("a file's object and a template reference each other, and the order comes from the references", async () => {
+    const { errors, doc } = await buildProject(mixed);
+    expect(errors).toEqual([]);
+    const objects = new Map(doc!.objects.map((o) => [o.export as string, o]));
+    // The file's table names the template's schema; the template's view names the file's table as text.
+    expect(objects.get("users")!.dependsOn).toEqual(["app"]);
+    expect(objects.get("notes")!.dependsOn).toEqual(["app", "users"]);
+    expect(objects.get("activeUsers")!.dependsOn).toEqual(["users"]);
+    expect(doc!.applyOrder).toEqual(["app", "users", "activeUsers", "notes"]);
+  });
+
+  test("a statement the build cannot read is a build error naming the file and the statement", async () => {
+    const dir = await project("bad", { "app.ts": `import { schema } from "@intentius/chant-lexicon-sql/postgres";\nexport const app = schema\`CREATE SCHEMA app\`;\n`, "bad.sql": "CREATE TABLE app.t (id int);\nDROP TABLE app.old;\n" });
+    dirs.push(dir);
+    const { errors } = await buildProject(dir);
+    expect(errors.join("\n")).toMatch(/src\/bad\.sql: a statement chant cannot read as declarations:\n {2}- not a statement that declares an object: DROP TABLE app\.old/);
+  });
+
+  test("an object declared in a file and a template both is an error naming both", async () => {
+    const dir = await project("twice", { "app.ts": `import { table } from "@intentius/chant-lexicon-sql/postgres";\nexport const accounts = table\`CREATE TABLE app.users (id int)\`;\n`, "users.sql": "CREATE TABLE app.users (id int);\n" });
+    dirs.push(dir);
+    const { errors } = await buildProject(dir);
+    expect(errors.join("\n")).toMatch(/app\.users is declared twice, here and as accounts/);
+  });
+
+  test("a file marked chant-discovery-skip is not read", async () => {
+    const dir = await project("skip", { "app.ts": `import { schema } from "@intentius/chant-lexicon-sql/postgres";\nexport const app = schema\`CREATE SCHEMA app\`;\n`, "seed.sql": "-- chant-discovery-skip\nINSERT INTO app.t VALUES (1);\n" });
+    dirs.push(dir);
+    const { errors, doc } = await buildProject(dir);
+    expect(errors).toEqual([]);
+    expect(doc!.applyOrder).toEqual(["app"]);
+  });
+});
+
+describe("references between a file's objects", () => {
+  test("a bare name where SQL names an object is a reference, and a cycle is an error", async () => {
+    const { sqlFileEntities } = await import("./sql-files");
+    const entities = sqlFileEntities("postgres", [
+      { origin: "a.sql", ddl: "CREATE TABLE orders (user_id int REFERENCES users (id));\nCREATE TABLE users (id int PRIMARY KEY);\nCREATE VIEW recent AS SELECT * FROM orders JOIN users ON true;" },
+    ]);
+    const deps = (n: string) => ((entities.get(n) as unknown as { dependsOn: Array<{ sqlName: string }> }).dependsOn).map((d) => d.sqlName);
+    expect(deps("orders")).toEqual(["users"]);
+    expect(deps("recent")).toEqual(["orders", "users"]);
+    expect(() => sqlFileEntities("postgres", [{ origin: "b.sql", ddl: "CREATE TABLE a (b int REFERENCES b (id), id int PRIMARY KEY);\nCREATE TABLE b (a int REFERENCES a (id), id int PRIMARY KEY);" }])).toThrow(/reference each other/);
+  });
+});

package/src/sql-files.ts

@@ -0,0 +1,218 @@
+/**
+ * Plain `.sql` files as a schema source (#3646): DDL read into the same
+ * declarations the tagged templates make. Also at the package root.
+ *
+ * - {@link readSqlFile}: the objects a file of DDL declares, each with its
+ *   export name, type, qualified name and DDL.
+ * - {@link sqlFileDeclarations}: the same file as TypeScript, one `schema.ts`
+ *   of `table`, `view`, ... templates, as `chant import <file>.sql` writes it.
+ * - {@link sqlFileEntities}: the declarations themselves, with the references
+ *   between the files' objects, and to objects declared elsewhere, as
+ *   interpolations, so dependency order comes from the graph. The build uses
+ *   it for every `.sql` file in the source directory.
+ *
+ * Postgres reads each `CREATE`, `GRANT`, `REVOKE` and `ALTER DEFAULT
+ * PRIVILEGES` as an object, with `COMMENT ON`, `ALTER TABLE ... ROW LEVEL
+ * SECURITY` and `ALTER TABLE ... ADD <table constraint>` folded into the
+ * object they finish; ClickHouse reads `CREATE DATABASE`, `TABLE`, `VIEW`,
+ * `MATERIALIZED VIEW`, `DICTIONARY` and `FUNCTION`. Any other statement, and any
+ * statement that does not parse, throws a {@link SqlFileError} naming every
+ * such statement: none is left out.
+ *
+ * A name is a reference where it is qualified (`app.users`), or bare where an
+ * object is named (`REFERENCES users`, `FROM users`, an index's table, a
+ * column of an enum type) and an object declared without a schema has it.
+ */
+
+import type { Declarable } from "@intentius/chant/declarable";
+import type { SqlDialect } from "./dialects";
+import { SqlFileError, type ReadOptions } from "./files/common";
+import { postgresTag, readPostgres, templateStrings } from "./files/postgres";
+import { clickhouseTag, readClickHouse } from "./files/clickhouse";
+import { exportNames as pgExportNames, objectsToIR as pgObjectsToIR, type ImportedPgObject } from "./postgres/import/ir";
+import { exportNames as chExportNames, objectsToIR as chObjectsToIR, type ImportedObject } from "./clickhouse/import/ir";
+import { PostgresGenerator, postgresTemplates, type PgTemplateItem, type TemplateBody } from "./postgres/import/generator";
+import { ClickHouseGenerator, clickhouseTemplates } from "./clickhouse/import/generator";
+import { isPostgresObject } from "./postgres/entities";
+import { isClickHouseObject } from "./clickhouse/entities";
+
+export { SqlFileError };
+export type { ReadOptions as SqlFileOptions };
+
+/** One object a file of DDL declares. */
+export interface SqlFileObject {
+  /** The export it is declared as. */
+  export: string;
+  /** Its entity type, `Postgres::Table`. */
+  type: string;
+  /** `schema.name` (Postgres), `database.name` (ClickHouse); the name alone when the DDL does not qualify it. */
+  name: string;
+  /** Its statements: the CREATE, then what was folded into it. */
+  ddl: string;
+  /** The file it came from, the `origin` it was read under. */
+  origin: string;
+}
+
+/** A file to read: its DDL and the name errors give it. */
+export interface SqlFileSource extends ReadOptions {
+  ddl: string;
+}
+
+type Imported = ImportedPgObject | ImportedObject;
+
+const qualifierOf = (o: Imported): string | undefined => ("schema" in o ? o.schema : "database" in o ? o.database : undefined);
+const qualifiedName = (o: Imported): string => (qualifierOf(o) ? `${qualifierOf(o)}.${o.name}` : o.name);
+
+function readObjects(dialect: SqlDialect, sources: readonly SqlFileSource[]): Array<Imported & { origin: string }> {
+  const out: Array<Imported & { origin: string }> = [];
+  for (const source of sources) {
+    const problems: string[] = [];
+    const objects = dialect === "postgres" ? readPostgres(source.ddl, source, problems) : readClickHouse(source.ddl, source, problems);
+    if (problems.length > 0) throw new SqlFileError(source.origin, problems);
+    out.push(...objects.map((o) => ({ ...o, origin: source.origin })));
+  }
+  return out;
+}
+
+/** Export names for objects read from files, unique across them. */
+function namesFor(dialect: SqlDialect, objects: readonly Imported[]): string[] {
+  return dialect === "postgres" ? pgExportNames(objects as ImportedPgObject[]) : chExportNames(objects as ImportedObject[]);
+}
+
+/** The objects a file of DDL declares. Throws a {@link SqlFileError} naming each statement it cannot read. */
+export function readSqlFile(dialect: SqlDialect, ddl: string, options: ReadOptions): SqlFileObject[] {
+  const objects = readObjects(dialect, [{ ...options, ddl }]);
+  const names = namesFor(dialect, objects);
+  return objects.map((o, i) => ({ export: names[i]!, type: o.type, name: qualifiedName(o), ddl: o.ddl, origin: o.origin }));
+}
+
+/**
+ * A file of DDL as TypeScript declarations: the `schema.ts` `chant import`
+ * writes, with `header` as comment lines at its top, and the objects in it.
+ */
+export function sqlFileDeclarations(dialect: SqlDialect, ddl: string, options: ReadOptions & { header?: string }): { content: string; objects: SqlFileObject[] } {
+  const objects = readObjects(dialect, [{ ...options, ddl }]);
+  if (objects.length === 0) throw new SqlFileError(options.origin, ["the DDL creates no objects"]);
+  const ir = dialect === "postgres" ? pgObjectsToIR(objects as ImportedPgObject[]) : chObjectsToIR(objects as ImportedObject[]);
+  const files = dialect === "postgres" ? new PostgresGenerator(options.header).generate(ir) : new ClickHouseGenerator(options.header).generate(ir);
+  const [file] = files;
+  if (!file) throw new SqlFileError(options.origin, ["the DDL creates no objects"]);
+  return {
+    content: file.content,
+    objects: ir.resources.map((r, i) => ({ export: r.logicalId, type: r.type, name: qualifiedName(objects[i]!), ddl: objects[i]!.ddl, origin: options.origin })),
+  };
+}
+
+/** Where an entity read from a `.sql` file came from: its `origin`. Undefined for any other entity. */
+export function sqlFileOf(entity: unknown): string | undefined {
+  return typeof entity === "object" && entity !== null ? ((entity as Record<symbol, unknown>)[FROM_FILE] as string | undefined) : undefined;
+}
+
+const FROM_FILE = Symbol.for("chant.sql.fromFile");
+
+/** A declared entity as a template target: its export, type and name. */
+function targetOf(dialect: SqlDialect, exportName: string, entity: Declarable): PgTemplateItem | undefined {
+  const p = (entity as unknown as { props: { schema?: string; database?: string; name?: string; ddl?: string } }).props;
+  if (typeof p.name !== "string") return undefined;
+  const qualifier = dialect === "postgres" ? p.schema : p.database;
+  return {
+    exportName,
+    type: entity.entityType,
+    ...(qualifier ? (dialect === "postgres" ? { schema: qualifier } : { database: qualifier }) : {}),
+    name: p.name,
+    ddl: typeof p.ddl === "string" ? p.ddl : "",
+  } as PgTemplateItem;
+}
+
+function templates(dialect: SqlDialect, items: readonly PgTemplateItem[], targets: readonly PgTemplateItem[]): { order: PgTemplateItem[]; bodies: Map<string, TemplateBody> } {
+  return dialect === "postgres" ? postgresTemplates(items, targets) : (clickhouseTemplates(items, targets) as { order: PgTemplateItem[]; bodies: Map<string, TemplateBody> });
+}
+
+/** The entities of one dialect in a set, by export name. */
+function ofDialect(dialect: SqlDialect, entities: ReadonlyMap<string, Declarable> | undefined): Map<string, Declarable> {
+  const is = dialect === "postgres" ? isPostgresObject : isClickHouseObject;
+  return new Map([...(entities ?? new Map<string, Declarable>())].filter(([, e]) => is(e)));
+}
+
+/**
+ * The declarations the files make, by export name. `known` holds the
+ * entities declared elsewhere (the project's tagged templates): a file's
+ * reference to one of them is an interpolation of it, as an import would
+ * write it, and an export name or an object one of them already declares is
+ * an error naming both.
+ */
+export function sqlFileEntities(dialect: SqlDialect, sources: readonly SqlFileSource[], options: { known?: ReadonlyMap<string, Declarable> } = {}): Map<string, Declarable> {
+  const objects = readObjects(dialect, sources);
+  const names = namesFor(dialect, objects);
+  const known = ofDialect(dialect, options.known);
+  const problems: string[] = [];
+
+  const items: PgTemplateItem[] = objects.map((o, i) => ({ exportName: names[i]!, type: o.type, ...("schema" in o && o.schema ? { schema: o.schema } : {}), ...("database" in o && o.database ? { database: o.database } : {}), name: o.name, ddl: o.ddl }) as PgTemplateItem);
+  const origin = new Map(objects.map((o, i) => [names[i]!, o.origin]));
+  const knownItems = [...known].map(([n, e]) => targetOf(dialect, n, e)).filter((t): t is PgTemplateItem => t !== undefined);
+  const declared = new Map(knownItems.map((t) => [`${t.type} ${qualifiedName(t as Imported)}`, t.exportName]));
+  const seen = new Map<string, string>();
+  const where = (at: string | undefined) => (sources.length > 1 && at ? `${at}: ` : "");
+  const label = sources.length === 1 ? sources[0]!.origin : "the .sql files";
+  for (const [i, it] of items.entries()) {
+    const what = `${it.type} ${qualifiedName(it as Imported)}`;
+    const at = objects[i]!.origin;
+    const q = qualifiedName(it as Imported);
+    if (options.known?.has(it.exportName)) problems.push(`${where(at)}${q} is exported as ${it.exportName}, a name another declaration already has; rename that one`);
+    const other = declared.get(what);
+    if (other) problems.push(`${where(at)}${q} is declared twice, here and as ${other}; leave it to one of them`);
+    const again = seen.get(what);
+    if (again) problems.push(`${where(at)}${q} is declared twice, here and in ${again}; leave it to one of them`);
+    seen.set(what, at);
+  }
+  if (problems.length > 0) throw new SqlFileError(label, problems);
+
+  const { order, bodies } = templates(dialect, items, [...items, ...knownItems]);
+  const built = new Map<string, Declarable>();
+  const failed = new Set<string>();
+  const tagOf = dialect === "postgres" ? postgresTag : clickhouseTag;
+  for (const it of order) {
+    const body = bodies.get(it.exportName)!;
+    const values = body.refs.map((r) => built.get(r) ?? known.get(r));
+    const missing = body.refs.find((_, i) => values[i] === undefined);
+    if (missing !== undefined) {
+      failed.add(it.exportName);
+      // What references an object that failed is not reported again.
+      if (failed.has(missing)) continue;
+      problems.push(`${where(origin.get(it.exportName))}${qualifiedName(it as Imported)} and ${missing} reference each other, which no declaration can; break the cycle`);
+      continue;
+    }
+    try {
+      const entity = tagOf(it.type)!(templateStrings(body.parts), ...values);
+      Object.defineProperty(entity, FROM_FILE, { value: origin.get(it.exportName), enumerable: false });
+      built.set(it.exportName, entity);
+    } catch (e) {
+      failed.add(it.exportName);
+      problems.push(`${where(origin.get(it.exportName))}${qualifiedName(it as Imported)}: ${e instanceof Error ? e.message : String(e)}`);
+    }
+  }
+  if (problems.length > 0) throw new SqlFileError(label, problems);
+  // In the files' order, not the dependency order.
+  return new Map(items.filter((it) => built.has(it.exportName)).map((it) => [it.exportName, built.get(it.exportName)!]));
+}
+
+/**
+ * The objects read from `.sql` files that each other object names in its
+ * DDL as text, by export name: a tagged template cannot import a file's
+ * object, so it names it, and this is the edge the dependency order needs.
+ * Empty when no object came from a file.
+ */
+export function sqlFileReferences(dialect: SqlDialect, objects: ReadonlyMap<string, Declarable>): Map<string, string[]> {
+  const fromFiles = [...objects].filter(([, e]) => sqlFileOf(e) !== undefined);
+  const out = new Map<string, string[]>();
+  if (fromFiles.length === 0) return out;
+  const targets = fromFiles.map(([n, e]) => targetOf(dialect, n, e)).filter((t): t is PgTemplateItem => t !== undefined);
+  for (const [name, entity] of objects) {
+    if (sqlFileOf(entity) !== undefined) continue;
+    const item = targetOf(dialect, name, entity);
+    if (!item || !item.ddl) continue;
+    const refs = templates(dialect, [item], targets).bodies.get(name)?.refs ?? [];
+    if (refs.length > 0) out.set(name, [...new Set(refs)]);
+  }
+  return out;
+}

package/src/token-source.e2e.test.ts

@@ -0,0 +1,142 @@
+/**
+ * Token sources against a live server (#3685). A `command` source stands in
+ * for the token endpoint: it prints a password from a file, and the test
+ * rotates the role's password and the file together, as a cloud rotates its
+ * tokens. A profile with that source connects; after the token's lifetime a
+ * new one is minted and the next connection uses it; a token the server
+ * refuses is forgotten, so the next attempt mints again.
+ *
+ * Postgres: `CHANT_SQL_E2E_POSTGRES_URL` (with
+ * `CHANT_SQL_E2E_POSTGRES_PASSWORD`) names a running server, else a
+ * throwaway one, skipped when Docker is not available. ClickHouse:
+ * `CHANT_SQL_E2E_CLICKHOUSE_URL` (as `default`, with
+ * `CHANT_SQL_E2E_CLICKHOUSE_PASSWORD`), skipped when not set. The test makes
+ * only the user `chant_e2e_3685` on each, dropped afterwards.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { dockerAvailable, startTestPostgres, type TestPostgres } from "./postgres/testing/server";
+import { connectPostgres, PostgresQueryError, type PostgresClient, type PostgresEndpoint } from "./postgres/live/client";
+import { resolveBoundTarget } from "./postgres/live/bind";
+import { bindClickHouse } from "./clickhouse/live/bind";
+import { clickhouseQuery, ClickHouseQueryError, type ClickHouseEndpoint } from "./clickhouse/http";
+
+const ROLE = "chant_e2e_3685";
+const dir = mkdtempSync(join(tmpdir(), "chant-token-"));
+const tokenFile = join(dir, "token");
+const mint = (password: string) => writeFileSync(tokenFile, password);
+const command = ["cat", tokenFile];
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+const pgGiven = process.env.CHANT_SQL_E2E_POSTGRES_URL;
+const pgEnabled = pgGiven ? true : await dockerAvailable();
+let server: TestPostgres | undefined;
+let admin: PostgresClient;
+let pgUrl: string;
+
+describe.skipIf(!pgEnabled)("a Postgres profile with a command token source", () => {
+  beforeAll(async () => {
+    let endpoint: PostgresEndpoint;
+    if (pgGiven) endpoint = { url: pgGiven, password: process.env.CHANT_SQL_E2E_POSTGRES_PASSWORD ?? "" };
+    else {
+      server = await startTestPostgres();
+      endpoint = server.endpoint();
+    }
+    admin = await connectPostgres(endpoint);
+    await admin.query(`DROP ROLE IF EXISTS ${ROLE}`);
+    await admin.query(`CREATE ROLE ${ROLE} LOGIN PASSWORD 'first-token'`);
+    const url = new URL(endpoint.url);
+    url.username = "";
+    url.password = "";
+    pgUrl = url.toString();
+  }, 600_000);
+
+  afterAll(async () => {
+    await admin?.query(`DROP ROLE IF EXISTS ${ROLE}`).catch(() => undefined);
+    await admin?.end();
+    await server?.stop();
+  });
+
+  const whoami = async (endpoint: PostgresEndpoint) => {
+    const client = await connectPostgres(endpoint);
+    try {
+      const [row] = await client.query<{ u: string }>("SELECT current_user AS u");
+      return row?.u;
+    } finally {
+      await client.end();
+    }
+  };
+
+  test("connects with the minted token, mints again after its lifetime, and forgets a refused one", async () => {
+    mint("first-token");
+    const config = { sql: { profiles: { e2e: { url: pgUrl, user: { env: "E2E_USER" }, password: { token: "command" as const, command, ttlSeconds: 1 } } } } };
+    const target = await resolveBoundTarget({ environment: "e2e", config, env: { ...process.env, E2E_USER: ROLE } });
+    expect(await whoami(target.endpoint)).toBe(ROLE);
+
+    // Rotated: the cached token has expired by the next connect, so the new one is minted.
+    await admin.query(`ALTER ROLE ${ROLE} PASSWORD 'second-token'`);
+    mint("second-token");
+    await sleep(1_000);
+    expect(await whoami(target.endpoint)).toBe(ROLE);
+
+    // Rotated before the cached token's lifetime is up: the server refuses it, and the retry mints.
+    const long = await resolveBoundTarget({
+      environment: "e2e",
+      config: { sql: { profiles: { e2e: { ...config.sql.profiles.e2e, password: { token: "command" as const, command, ttlSeconds: 3600 } } } } },
+      env: { ...process.env, E2E_USER: ROLE },
+    });
+    expect(await whoami(long.endpoint)).toBe(ROLE);
+    await admin.query(`ALTER ROLE ${ROLE} PASSWORD 'third-token'`);
+    mint("third-token");
+    const refused = await whoami(long.endpoint).catch((e: unknown) => e);
+    expect(refused).toBeInstanceOf(PostgresQueryError);
+    expect((refused as PostgresQueryError).code).toBe("28P01");
+    expect(await whoami(long.endpoint)).toBe(ROLE);
+  }, 60_000);
+});
+
+const chGiven = process.env.CHANT_SQL_E2E_CLICKHOUSE_URL;
+
+describe.skipIf(!chGiven)("a ClickHouse profile with a command token source", () => {
+  const admin: ClickHouseEndpoint = { url: chGiven ?? "", user: "default", password: process.env.CHANT_SQL_E2E_CLICKHOUSE_PASSWORD ?? "" };
+
+  beforeAll(async () => {
+    await clickhouseQuery(admin, `DROP USER IF EXISTS ${ROLE}`);
+    await clickhouseQuery(admin, `CREATE USER ${ROLE} IDENTIFIED BY 'first-token'`);
+  });
+
+  afterAll(async () => {
+    await clickhouseQuery(admin, `DROP USER IF EXISTS ${ROLE}`).catch(() => undefined);
+  });
+
+  test("each request carries a valid token; a refused one is minted again", async () => {
+    mint("first-token");
+    const config = { sql: { profiles: { e2e: { url: chGiven!, user: { env: "E2E_USER" }, password: { token: "command" as const, command, ttlSeconds: 1 } } } } };
+    const target = await bindClickHouse({ environment: "e2e", config, env: { ...process.env, E2E_USER: ROLE } });
+    const whoami = async () => (await clickhouseQuery<{ u: string }>(target.endpoint, "SELECT currentUser() AS u"))[0]?.u;
+    expect(await whoami()).toBe(ROLE);
+
+    await clickhouseQuery(admin, `ALTER USER ${ROLE} IDENTIFIED BY 'second-token'`);
+    mint("second-token");
+    await sleep(1_000);
+    expect(await whoami()).toBe(ROLE);
+
+    // Rotated before the cached token's lifetime is up: refused once, then minted again.
+    const long = await bindClickHouse({
+      environment: "e2e",
+      config: { sql: { profiles: { e2e: { ...config.sql.profiles.e2e, password: { token: "command" as const, command, ttlSeconds: 3600 } } } } },
+      env: { ...process.env, E2E_USER: ROLE },
+    });
+    const longWhoami = async () => (await clickhouseQuery<{ u: string }>(long.endpoint, "SELECT currentUser() AS u"))[0]?.u;
+    expect(await longWhoami()).toBe(ROLE);
+    await clickhouseQuery(admin, `ALTER USER ${ROLE} IDENTIFIED BY 'third-token'`);
+    mint("third-token");
+    await expect(longWhoami()).rejects.toBeInstanceOf(ClickHouseQueryError);
+    expect(await longWhoami()).toBe(ROLE);
+  }, 60_000);
+});
+
+afterAll(() => rmSync(dir, { recursive: true, force: true }));