@intentius/chant-lexicon-sql 0.124.11 → 0.124.13

package/src/read/postgres.ts

@@ -0,0 +1,180 @@
+/**
+ * The Postgres half of the public read API (`../read.ts`): the build's
+ * tables, columns, keys, indexes and views as typed values, built from the
+ * serialized output. The lexicon's post-synth checks read the same values
+ * (`../lint/post-synth/postgres-helpers.ts`).
+ */
+
+import { POSTGRES_MAJORS } from "../generated/postgres";
+import { relationKey } from "../core/typed";
+import { parseType, projectFromObjects, resolveType, typeLabel, type PgProject } from "../lint/post-synth/postgres-names";
+import {
+  dialectObjects,
+  recordedPostgresMajor,
+  type PostgresIndexJson,
+  type PostgresTableJson,
+  type PostgresViewJson,
+  type SqlObjectJson,
+  type SqlReadSource,
+} from "./objects";
+import type { ColumnDef } from "../postgres/entities";
+
+/** A column of a Postgres table, as declared, with what follows from the declaration. */
+export interface PostgresColumnInfo extends ColumnDef {
+  /**
+   * Whether the column can hold NULL: false when it is `NOT NULL`, in the
+   * primary key, an identity column or a `serial`, true otherwise.
+   */
+  nullable: boolean;
+  /**
+   * The type's canonical name, without modifiers or array brackets: a
+   * built-in as Postgres's `format_type()` spells it (`timestamptz` is
+   * `timestamp with time zone`, `int8` and `bigserial` are `bigint`), a type
+   * the project declares by its SQL name, anything else as written, lower
+   * case. Undefined when the column has no type of its own (a partition, a
+   * typed table) or the type is not a plain name.
+   */
+  typeName?: string;
+  /** `text[]`, `integer ARRAY`. */
+  array: boolean;
+  /** One of the primary key's columns. */
+  inPrimaryKey: boolean;
+}
+
+/** A `CREATE INDEX`, with its table's export name and the columns it leads with. */
+export interface PostgresIndexInfo extends PostgresIndexJson {
+  /** The elements that are bare columns, in order, up to the first expression. */
+  columns: string[];
+}
+
+/** A Postgres table, as declared: its columns, keys, constraints and the indexes declared on it. */
+export interface PostgresTableInfo extends Omit<PostgresTableJson, "columns"> {
+  /** `schema.name` when the DDL qualifies it, else the name: the table's key in `SqlSchema`. */
+  qualifiedName: string;
+  columns: PostgresColumnInfo[];
+  /** The `CREATE INDEX` statements on this table (constraints' own indexes are `primaryKey` and `uniques`). */
+  indexes: PostgresIndexInfo[];
+  /** The column named `name`, if the table declares one. */
+  column(name: string): PostgresColumnInfo | undefined;
+}
+
+/** A view or materialized view, with the names of its output columns. */
+export interface PostgresViewInfo extends PostgresViewJson {
+  qualifiedName: string;
+  materialized: boolean;
+  /** The declared column list, else the outputs of the query's select list. */
+  outputs: string[];
+}
+
+/** What a Postgres build declares. */
+export interface PostgresRead {
+  dialect: "postgres";
+  /** The major the build targets. */
+  major: number;
+  tables: PostgresTableInfo[];
+  views: PostgresViewInfo[];
+  indexes: PostgresIndexInfo[];
+  /** Every serialized object, sequences, functions, roles and grants included, in apply order. */
+  objects: SqlObjectJson[];
+  /** A table by export name, `schema.name`, or name. */
+  table(ref: string): PostgresTableInfo | undefined;
+}
+
+const PINNED_MAJOR: number = POSTGRES_MAJORS[POSTGRES_MAJORS.length - 1]!;
+
+/** A column type's canonical name, array-ness and serial-ness against a project. */
+function typeOf(p: PgProject, text: string | undefined): { typeName?: string; array: boolean; serial: boolean } {
+  const t = parseType(text);
+  if (!t) return { ...(text ? { typeName: text.trim().toLowerCase() } : {}), array: false, serial: false };
+  const r = resolveType(p, t, { serial: true });
+  const typeName =
+    r.kind === "builtin" || r.kind === "absent" ? r.canonical : r.kind === "declared" ? String(r.object.sqlName ?? r.object.name) : typeLabel(t);
+  return { typeName, array: t.array, serial: r.kind === "builtin" && r.serial === true };
+}
+
+/**
+ * The canonical name of a Postgres type as written: `timestamptz` and
+ * `TIMESTAMP WITH TIME ZONE` are both `timestamp with time zone`, `int4` is
+ * `integer`. Compare a column's {@link PostgresColumnInfo.typeName} with it,
+ * so a rule does not depend on how the DDL spelled the type.
+ */
+export function canonicalPostgresType(text: string): string | undefined {
+  return typeOf(projectFromObjects([], PINNED_MAJOR), text).typeName;
+}
+
+function columnInfo(p: PgProject, c: ColumnDef, pk: ReadonlySet<string>): PostgresColumnInfo {
+  const { typeName, array, serial } = typeOf(p, c.type);
+  const inPrimaryKey = pk.has(c.name);
+  const nullable = !(c.notNull === true || inPrimaryKey || c.generated?.kind === "identity" || serial);
+  return { ...c, nullable, ...(typeName !== undefined ? { typeName } : {}), array, inPrimaryKey };
+}
+
+function indexInfo(i: PostgresIndexJson): PostgresIndexInfo {
+  const columns: string[] = [];
+  for (const e of i.elements ?? []) {
+    if (!e.column) break;
+    columns.push(e.column);
+  }
+  return { ...i, columns };
+}
+
+function build(objects: SqlObjectJson[], major: number): PostgresRead {
+  const p = projectFromObjects(objects, major);
+  const indexes = objects.filter((o) => o.type === "Postgres::Index").map((o) => indexInfo(o as unknown as PostgresIndexJson));
+  const tables = objects
+    .filter((o) => o.type === "Postgres::Table")
+    .map((o): PostgresTableInfo => {
+      const t = o as unknown as PostgresTableJson;
+      const pk = new Set(t.primaryKey?.columns ?? []);
+      const columns = (t.columns ?? []).map((c) => columnInfo(p, c, pk));
+      return {
+        ...t,
+        qualifiedName: relationKey(t),
+        columns,
+        indexes: indexes.filter((i) => i.table === t.export || i.tableName === t.sqlName),
+        column: (name) => columns.find((c) => c.name === name),
+      };
+    });
+  const views = objects
+    .filter((o) => o.type === "Postgres::View" || o.type === "Postgres::MaterializedView")
+    .map((o): PostgresViewInfo => {
+      const v = o as unknown as PostgresViewJson;
+      const declared = v.columns ?? [];
+      return {
+        ...v,
+        qualifiedName: relationKey(v),
+        materialized: v.type === "Postgres::MaterializedView",
+        outputs: declared.length > 0 ? [...declared] : (v.lineage ?? []).map((e) => e.output),
+      };
+    });
+  return {
+    dialect: "postgres",
+    major,
+    tables,
+    views,
+    indexes,
+    objects,
+    table: (ref) => tables.find((t) => t.export === ref) ?? tables.find((t) => t.qualifiedName === ref) ?? tables.find((t) => t.name === ref),
+  };
+}
+
+const cache = new WeakMap<object, PostgresRead>();
+
+/**
+ * What a Postgres build declares, read from a post-synth context's `docs`
+ * (a `lint.policies` check's `ctx`), or from the serialized output document
+ * itself. Empty when the build has no Postgres output. The result is cached
+ * per context; treat it as read-only.
+ */
+export function readPostgres(source: SqlReadSource | { dialect: string; objects: unknown[]; postgresMajor?: number }): PostgresRead {
+  if ("objects" in source) {
+    const objects = source.dialect === "postgres" ? (source.objects as SqlObjectJson[]) : [];
+    return build(objects, typeof source.postgresMajor === "number" ? source.postgresMajor : PINNED_MAJOR);
+  }
+  const key = source.docs;
+  const hit = key ? cache.get(key) : undefined;
+  if (hit) return hit;
+  const read = build(dialectObjects(source, "postgres"), recordedPostgresMajor(source, PINNED_MAJOR));
+  if (key) cache.set(key, read);
+  return read;
+}

package/src/read-policies.e2e.test.ts

@@ -0,0 +1,144 @@
+/**
+ * #3870: a project's own `lint.policies`, written against
+ * `@intentius/chant-lexicon-sql/read`, fail `chant lint` and `chant build` on
+ * a table that breaks them, and `tsc` checks the policy itself.
+ *
+ * Each case copies `examples/postgres-policies` into a temporary project that
+ * reaches the workspace packages through a `node_modules` link, as an
+ * installed project would.
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from "vitest";
+import { cpSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { lintCommand } from "@intentius/chant/cli/commands/lint";
+import { buildCommand } from "@intentius/chant/cli/commands/build";
+import { ts } from "@intentius/chant/lint/typescript";
+import { sqlPlugin, sqlSerializer } from "@intentius/chant-lexicon-sql";
+
+const here = import.meta.dirname;
+const example = resolve(here, "..", "examples", "postgres-policies");
+const repoNodeModules = resolve(here, "..", "..", "..", "node_modules");
+
+let root: string;
+
+beforeEach(() => {
+  root = realpathSync(mkdtempSync(join(tmpdir(), "chant-3870-")));
+  for (const f of ["chant.config.ts", "policies", "src"]) cpSync(join(example, f), join(root, f), { recursive: true });
+  symlinkSync(repoNodeModules, join(root, "node_modules"), "dir");
+  process.env.NO_COLOR = "1";
+});
+
+afterEach(() => {
+  rmSync(root, { recursive: true, force: true });
+  delete process.env.NO_COLOR;
+});
+
+/** Drop `sessions.created_at` from the example's schema. */
+function breakSessions(): void {
+  const file = join(root, "src", "app.ts");
+  const text = readFileSync(file, "utf-8").replace(
+    "    account_id bigint NOT NULL REFERENCES ${accounts} (id),\n    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()\n",
+    "    account_id bigint NOT NULL REFERENCES ${accounts} (id)\n",
+  );
+  expect(text).not.toContain("TIMESTAMP WITH TIME ZONE");
+  writeFileSync(file, text);
+}
+
+const policyIds = (diags: Array<{ ruleId: string }>) => diags.map((d) => d.ruleId).filter((id) => id.startsWith("ORG-"));
+
+describe("project policies over the sql read API (#3870)", () => {
+  test("chant lint: the example is clean, and a table without created_at fails", async () => {
+    const clean = await lintCommand({ path: join(root, "src"), format: "stylish" });
+    expect(policyIds(clean.diagnostics), clean.output).toEqual([]);
+
+    breakSessions();
+    const result = await lintCommand({ path: join(root, "src"), format: "stylish" });
+    expect(result.success).toBe(false);
+    const diag = result.diagnostics.find((d) => d.ruleId === "ORG-CREATED-AT");
+    expect(diag, result.output).toBeDefined();
+    expect(diag!.severity).toBe("error");
+    expect(diag!.message).toContain("app.sessions has no created_at column");
+    expect(diag!.file).toBe(join(root, "src", "app.ts"));
+  });
+
+  test("chant lint applies lint.rules to a policy's id", async () => {
+    breakSessions();
+    writeFileSync(
+      join(root, "chant.config.ts"),
+      readFileSync(join(root, "chant.config.ts"), "utf-8").replace(
+        'lint: { policies: ["policies/columns.ts"] }',
+        'lint: { policies: ["policies/columns.ts"], rules: { "ORG-CREATED-AT": "off" } }',
+      ),
+    );
+    const result = await lintCommand({ path: join(root, "src"), format: "stylish" });
+    expect(policyIds(result.diagnostics), result.output).toEqual([]);
+  });
+
+  test("chant build fails on the same table", async () => {
+    breakSessions();
+    const result = await buildCommand({ path: join(root, "src"), format: "json", serializers: [sqlSerializer], plugins: [sqlPlugin] });
+    expect(result.success).toBe(false);
+    expect(result.errors.join("\n")).toContain("[policy:ORG-CREATED-AT] [sessions]");
+  });
+
+  // The first case above lints the same files with no env and gets no ORG-COLUMN-COMMENTS finding.
+  // (The config is written before any lint here: a loaded chant.config.ts is cached for the process.)
+  test("an env-aware policy fires in prod", async () => {
+    writeFileSync(
+      join(root, "chant.config.ts"),
+      readFileSync(join(root, "chant.config.ts"), "utf-8").replace('sql: { dialect: "postgres" },', 'sql: { dialect: "postgres" },\n  ownership: { env: "prod" },'),
+    );
+    const prod = await lintCommand({ path: join(root, "src"), format: "stylish" });
+    expect(prod.diagnostics.filter((d) => d.ruleId === "ORG-COLUMN-COMMENTS").map((d) => d.message), prod.output).toEqual([
+      "[accounts] app.accounts.created_at has no comment",
+      "[sessions] app.sessions.account_id has no comment",
+      "[sessions] app.sessions.created_at has no comment",
+    ]);
+  });
+
+  test("the ClickHouse policy flags a MergeTree table without created_at, and a Nullable sort key column", async () => {
+    rmSync(join(root, "src"), { recursive: true });
+    mkdirSync(join(root, "src"));
+    writeFileSync(
+      join(root, "chant.config.ts"),
+      'import type { ChantConfig } from "@intentius/chant";\n' +
+        'export default { lexicons: ["sql"], sql: { dialect: "clickhouse" }, lint: { policies: ["policies/clickhouse.ts"], rules: { SQLCH113: "off" } } } satisfies ChantConfig;\n',
+    );
+    writeFileSync(
+      join(root, "src", "schema.ts"),
+      'import { database, table } from "@intentius/chant-lexicon-sql/clickhouse";\n' +
+        "export const shop = database`CREATE DATABASE shop ENGINE = Atomic`;\n" +
+        "export const events = table`CREATE TABLE ${shop}.events (id UInt64, created_at DateTime) ENGINE = MergeTree ORDER BY id`;\n" +
+        "export const visits = table`CREATE TABLE ${shop}.visits (id UInt64, ref Nullable(String)) ENGINE = MergeTree ORDER BY (id, ref) SETTINGS allow_nullable_key = 1`;\n",
+    );
+    const result = await lintCommand({ path: join(root, "src"), format: "stylish" });
+    expect(result.diagnostics.filter((d) => d.ruleId === "ORG-CH-CREATED-AT").map((d) => d.message), result.output).toEqual([
+      "[visits] shop.visits needs created_at DateTime, not Nullable",
+      "[visits] shop.visits.ref is Nullable and in the sort key",
+    ]);
+  });
+
+  test("tsc checks a policy against the read API's types", () => {
+    const policy = join(root, "policies", "columns.ts");
+    const check = () => {
+      const program = ts.createProgram([policy], {
+        target: ts.ScriptTarget.ES2022,
+        module: ts.ModuleKind.ESNext,
+        moduleResolution: ts.ModuleResolutionKind.Bundler,
+        strict: true,
+        noEmit: true,
+        skipLibCheck: true,
+        types: [],
+      });
+      return ts
+        .getPreEmitDiagnostics(program)
+        .filter((d) => d.file?.fileName === policy)
+        .map((d) => ts.flattenDiagnosticMessageText(d.messageText, "\n"));
+    };
+    expect(check()).toEqual([]);
+    writeFileSync(policy, readFileSync(policy, "utf-8").replace("c.typeName === TIMESTAMPTZ", "c.typeNmae === TIMESTAMPTZ"));
+    expect(check().join("\n")).toMatch(/Property 'typeNmae' does not exist on type 'PostgresColumnInfo'/);
+  }, 60_000);
+});

package/src/read.test.ts

@@ -0,0 +1,143 @@
+import { describe, expect, test } from "vitest";
+import { makePostSynthCtx } from "@intentius/chant-test-utils";
+import type { PostSynthCheck, PostSynthDiagnostic } from "@intentius/chant/lint/post-synth";
+import type { SerializerResult } from "@intentius/chant/serializer";
+import { sqlSerializer } from "./serializer";
+import { database, table as chTable, view as chView } from "./clickhouse/entities";
+import { index, schema, table, view } from "./postgres/entities";
+import { canonicalPostgresType, readClickHouse, readPostgres, readSql } from "./read";
+import { readPostgres as fromPostgresSubpath } from "./postgres";
+import { readClickHouse as fromClickHouseSubpath } from "./clickhouse";
+import { tablesOf as pgTablesOf } from "./lint/post-synth/postgres-helpers";
+import { tablesOf as chTablesOf } from "./lint/post-synth/clickhouse-helpers";
+
+const ctxOf = (entities: Record<string, unknown>) => {
+  const out = sqlSerializer.serialize(new Map(Object.entries(entities)) as never) as SerializerResult;
+  return makePostSynthCtx("sql", out.primary);
+};
+
+describe("readPostgres (#3870)", () => {
+  const app = schema`CREATE SCHEMA app`;
+  const users = table`
+    CREATE TABLE ${app}.users (
+      id bigserial PRIMARY KEY,
+      email text NOT NULL UNIQUE,
+      nickname varchar(40) DEFAULT 'anon',
+      tags text[],
+      created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
+    )`;
+  const orders = table`
+    CREATE TABLE ${app}.orders (
+      id bigint GENERATED ALWAYS AS IDENTITY,
+      user_id bigint NOT NULL REFERENCES ${users} (id),
+      created_at timestamp,
+      PRIMARY KEY (id)
+    )`;
+  const ordersUser = index`CREATE INDEX orders_user_id ON ${orders} (user_id, (lower('x')))`;
+  const totals = view`CREATE VIEW ${app}.totals AS SELECT user_id, count(*) AS n FROM ${orders} GROUP BY user_id`;
+  const ctx = ctxOf({ app, users, orders, ordersUser, totals });
+
+  test("tables carry typed columns: canonical type, nullability, default, keys", () => {
+    const db = readPostgres(ctx);
+    expect(db.dialect).toBe("postgres");
+    expect(db.tables.map((t) => t.qualifiedName).sort()).toEqual(["app.orders", "app.users"]);
+    const u = db.table("users")!;
+    expect(u.column("id")).toMatchObject({ typeName: "bigint", nullable: false, inPrimaryKey: true });
+    expect(u.column("email")).toMatchObject({ typeName: "text", nullable: false, inPrimaryKey: false });
+    expect(u.column("nickname")).toMatchObject({ typeName: "character varying", nullable: true, default: "'anon'" });
+    expect(u.column("tags")).toMatchObject({ typeName: "text", array: true });
+    expect(u.column("created_at")).toMatchObject({ typeName: "timestamp with time zone", nullable: false, default: "now()" });
+    expect(u.uniques[0]!.columns).toEqual(["email"]);
+    expect(db.table("app.orders")!.column("id")).toMatchObject({ nullable: false, inPrimaryKey: true });
+    expect(db.table("orders")!.column("created_at")).toMatchObject({ typeName: "timestamp without time zone", nullable: true });
+    expect(db.table("orders")!.foreignKeys[0]).toMatchObject({ columns: ["user_id"], references: "users" });
+  });
+
+  test("indexes belong to their table and lead with their bare columns", () => {
+    const db = readPostgres(ctx);
+    expect(db.indexes).toHaveLength(1);
+    expect(db.indexes[0]).toMatchObject({ export: "ordersUser", table: "orders", columns: ["user_id"] });
+    expect(db.table("orders")!.indexes.map((i) => i.export)).toEqual(["ordersUser"]);
+  });
+
+  test("views name their outputs and keep their lineage", () => {
+    const v = readPostgres(ctx).views[0]!;
+    expect(v).toMatchObject({ qualifiedName: "app.totals", materialized: false, outputs: ["user_id", "n"] });
+    expect(v.lineage.map((e) => e.output)).toEqual(["user_id", "n"]);
+  });
+
+  test("canonicalPostgresType spells aliases the way typeName does", () => {
+    expect(canonicalPostgresType("timestamptz")).toBe("timestamp with time zone");
+    expect(canonicalPostgresType("INT4")).toBe("integer");
+    expect(canonicalPostgresType("varchar(10)")).toBe("character varying");
+  });
+
+  test("the checks read the same values, and the dialect subpath exports the same reader", () => {
+    expect(pgTablesOf(ctx)).toBe(readPostgres(ctx).tables);
+    expect(fromPostgresSubpath).toBe(readPostgres);
+    expect(readClickHouse(ctx).tables).toEqual([]);
+    expect(readSql(ctx)?.dialect).toBe("postgres");
+  });
+
+  test("reads a serialized document as well as a context", () => {
+    const doc = JSON.parse((sqlSerializer.serialize(new Map(Object.entries({ app, users })) as never) as SerializerResult).primary) as { dialect: string; objects: unknown[] };
+    expect(readPostgres(doc).table("users")!.column("email")!.nullable).toBe(false);
+    expect(readSql(doc as never)?.dialect).toBe("postgres");
+  });
+
+  test("a project policy written against the read API flags a table without created_at timestamptz NOT NULL", () => {
+    const TIMESTAMPTZ = canonicalPostgresType("timestamptz");
+    const policy: PostSynthCheck = {
+      id: "ORG-CREATED-AT",
+      description: "every table has created_at timestamptz NOT NULL",
+      check(c): PostSynthDiagnostic[] {
+        return readPostgres(c)
+          .tables.filter((t) => {
+            const col = t.column("created_at");
+            return !col || col.typeName !== TIMESTAMPTZ || col.nullable;
+          })
+          .map((t) => ({ checkId: "ORG-CREATED-AT", severity: "error", entity: t.export, message: `${t.qualifiedName} needs created_at timestamptz NOT NULL` }));
+      },
+    };
+    expect(policy.check(ctx).map((d) => d.entity)).toEqual(["orders"]);
+  });
+});
+
+describe("readClickHouse (#3870)", () => {
+  const shop = database`CREATE DATABASE shop ENGINE = Atomic`;
+  const events = chTable`
+    CREATE TABLE ${shop}.events (
+      ts DateTime64(3) CODEC(Delta, ZSTD),
+      kind LowCardinality(String),
+      user_id UInt64,
+      ref Nullable(String) COMMENT 'referrer',
+      label LowCardinality(Nullable(String)) DEFAULT 'x',
+      INDEX kind_idx kind TYPE set(100) GRANULARITY 4
+    )
+    ENGINE = ReplicatedReplacingMergeTree('/ch/{shard}/events', '{replica}', ts)
+    ORDER BY (kind, toDate(ts), user_id)`;
+  const daily = chView`CREATE VIEW ${shop}.daily AS SELECT toDate(ts) AS day, count() AS n FROM ${events} GROUP BY day`;
+  const ctx = ctxOf({ shop, events, daily });
+
+  test("tables carry typed columns, the engine family and the keys", () => {
+    const db = readClickHouse(ctx);
+    const t = db.table("shop.events")!;
+    expect(t).toMatchObject({ qualifiedName: "shop.events", engineFamily: "ReplacingMergeTree", mergeTree: true });
+    expect(t.sortKey).toEqual(["kind", "toDate(ts)", "user_id"]);
+    expect(t.primaryKeyElements).toEqual(t.sortKey);
+    expect(t.column("ts")).toMatchObject({ family: "DateTime64", nullable: false, inSortKey: false, codec: "Delta, ZSTD" });
+    expect(t.column("kind")).toMatchObject({ lowCardinality: true, baseType: "String", nullable: false, inSortKey: true });
+    expect(t.column("ref")).toMatchObject({ nullable: true, comment: "referrer" });
+    expect(t.column("label")).toMatchObject({ nullable: true, lowCardinality: true, default: { kind: "DEFAULT", expr: "'x'" } });
+    expect(t.indexes[0]).toMatchObject({ name: "kind_idx", type: "set(100)" });
+  });
+
+  test("views name their outputs; the checks and the subpath share the reader", () => {
+    const v = readClickHouse(ctx).views[0]!;
+    expect(v).toMatchObject({ materialized: false, outputs: ["day", "n"] });
+    expect(chTablesOf(ctx)).toBe(readClickHouse(ctx).tables);
+    expect(fromClickHouseSubpath).toBe(readClickHouse);
+    expect(readSql(ctx)?.dialect).toBe("clickhouse");
+    expect(readPostgres(ctx).tables).toEqual([]);
+  });
+});

package/src/read.ts

@@ -0,0 +1,69 @@
+/**
+ * `@intentius/chant-lexicon-sql/read`: a typed, read-only view of the SQL
+ * objects a build declares (#3870), for a project's own `lint.policies`.
+ *
+ * A policy's `check(ctx)` gets the build's parsed output in `ctx.docs`.
+ * {@link readPostgres} and {@link readClickHouse} turn the sql lexicon's part
+ * of it into tables with typed columns (SQL type, nullability, default,
+ * comment), keys, indexes and engines, and views with their outputs and
+ * column lineage. The lexicon's own post-synth checks read the same values,
+ * so a project rule sees exactly what `SQLPG101` or `SQLCH113` sees.
+ *
+ * ```ts
+ * import type { PostSynthCheck } from "@intentius/chant/lint/post-synth";
+ * import { canonicalPostgresType, readPostgres } from "@intentius/chant-lexicon-sql/read";
+ *
+ * const TIMESTAMPTZ = canonicalPostgresType("timestamptz");
+ *
+ * export const createdAt: PostSynthCheck = {
+ *   id: "ORG-CREATED-AT",
+ *   description: "every table has created_at timestamptz NOT NULL",
+ *   check(ctx) {
+ *     return readPostgres(ctx).tables
+ *       .filter((t) => { const c = t.column("created_at"); return !c || c.typeName !== TIMESTAMPTZ || c.nullable; })
+ *       .map((t) => ({ checkId: "ORG-CREATED-AT", severity: "error", entity: t.export,
+ *                      message: `${t.qualifiedName} needs created_at timestamptz NOT NULL` }));
+ *   },
+ * };
+ * ```
+ *
+ * Every value keeps the fields the serializer writes (see the Serialization
+ * page) and adds the derived ones documented on each type. The reads also take
+ * the serialized document itself (`{ dialect, objects }`), for a tool that
+ * holds a recorded build rather than a post-synth context.
+ */
+
+export { readPostgres, canonicalPostgresType } from "./read/postgres";
+export type { PostgresRead, PostgresTableInfo, PostgresColumnInfo, PostgresIndexInfo, PostgresViewInfo } from "./read/postgres";
+export { readClickHouse } from "./read/clickhouse";
+export type { ClickHouseRead, ClickHouseTableInfo, ClickHouseColumnInfo, ClickHouseViewInfo } from "./read/clickhouse";
+export { sqlDocuments, isSqlOutputDocument } from "./read/objects";
+export type {
+  SqlReadSource,
+  SqlOutputDocument,
+  SqlObjectBase,
+  SqlObjectJson,
+  RefJson,
+  LineageJson,
+  PostgresTableJson,
+  PostgresIndexJson,
+  PostgresViewJson,
+  PostgresForeignKeyJson,
+  ClickHouseTableJson,
+  ClickHouseViewJson,
+} from "./read/objects";
+
+import { readClickHouse, type ClickHouseRead } from "./read/clickhouse";
+import { readPostgres, type PostgresRead } from "./read/postgres";
+import { sqlDocuments, type SqlOutputDocument, type SqlReadSource } from "./read/objects";
+
+/**
+ * The build's SQL objects in whichever dialect it declares: one build holds
+ * one dialect. Undefined when the build has no sql output.
+ */
+export function readSql(source: SqlReadSource | SqlOutputDocument): PostgresRead | ClickHouseRead | undefined {
+  const dialect = "objects" in source ? source.dialect : sqlDocuments(source)[0]?.dialect;
+  if (dialect === "postgres") return readPostgres(source);
+  if (dialect === "clickhouse") return readClickHouse(source);
+  return undefined;
+}

package/src/serializer.ts

@@ -25,6 +25,9 @@
  * targets (`sql.postgresMajor`, else the newest pinned major). Post-synth
  * checks and the lock classifier read it; ClickHouse output has no such field.
  *
+ * A clause chant's parser does not know (#3872) is kept in the statement as
+ * written and named in a build warning ({@link unparsedWarnings}).
+ *
  * Rule ids: `SQL` for rules that hold in every dialect, `SQLCH` for the
  * ClickHouse dialect's (#3199), `SQLPG` for Postgres's (#3289).
  */
@@ -36,10 +39,39 @@ import { isClickHouseObject, type ClickHouseObject } from "./clickhouse/entities
 import { isPostgresObject, type PostgresObject } from "./postgres/entities";
 import { POSTGRES_LATEST_MAJOR } from "./spec/postgres-pin";
 import { applyOrder, lineageJson, referenceName, type LineageEdge } from "./core/references";
-import { sqlFileReferences } from "./sql-files";
+import { sqlFileLineOf, sqlFileOf, sqlFileReferences } from "./sql-files";
+import { relative } from "node:path";
+import { getProvenance } from "@intentius/chant/provenance";
+import type { UnparsedClause } from "./core/interpolation";
 
 export { applyOrder } from "./core/references";
 
+/**
+ * A warning for each clause chant's parser does not know (#3872): where it
+ * is (the `.sql` file and line, or the template's file and the line in its
+ * statement), which object, and what follows from it. The statement is sent
+ * as written; the server decides.
+ */
+export function unparsedWarnings(objects: ReadonlyMap<string, Declarable>): string[] {
+  const out: string[] = [];
+  for (const [name, entity] of objects) {
+    const unparsed = (entity as unknown as { props?: { unparsed?: UnparsedClause[] } }).props?.unparsed;
+    if (!unparsed || unparsed.length === 0) continue;
+    const file = sqlFileOf(entity);
+    const start = sqlFileLineOf(entity);
+    const source = getProvenance(entity)?.sourceFile;
+    for (const u of unparsed) {
+      const where = file !== undefined ? `${file}:${start !== undefined ? start + u.line - 1 : u.line}` : `${source ? relative(process.cwd(), source) : "?"}: ${name}, line ${u.line} of its statement`;
+      const on = u.column !== undefined ? ` on column ${u.column}` : "";
+      out.push(
+        `${where}: ${(entity as { sqlName?: string }).sqlName ?? name} has a clause chant does not read, ${u.clause}${on}. ` +
+          "It is sent as written and the server checks it; a plan reads the rest of the object, and classes any change to the clause as one to review",
+      );
+    }
+  }
+  return out;
+}
+
 /** The file the statements are written to, beside the primary output. */
 export const CLICKHOUSE_DDL_FILE = "clickhouse.sql";
 
@@ -104,10 +136,12 @@ function serializePostgres(
     objects: order.map((n) => postgresObjectJson(n, objects.get(n)!, names, extra.get(n))),
   };
   const ddl = order.map((n) => `${(objects.get(n)!.props as { ddl: string }).ddl};`).join("\n\n");
+  const warnings = unparsedWarnings(objects);
   return {
     primary: `${JSON.stringify(doc, null, 2)}\n`,
     files: { [POSTGRES_DDL_FILE]: `${ddl}\n` },
     verbatimFiles: [POSTGRES_DDL_FILE],
+    ...(warnings.length > 0 ? { warnings } : {}),
   };
 }
 
@@ -144,10 +178,12 @@ export const sqlSerializer: Serializer = {
     const ddl = order
       .map((n) => `${(objects.get(n)!.props as { ddl: string }).ddl};`)
       .join("\n\n");
+    const warnings = unparsedWarnings(objects);
     return {
       primary: `${JSON.stringify(doc, null, 2)}\n`,
       files: { [CLICKHOUSE_DDL_FILE]: `${ddl}\n` },
       verbatimFiles: [CLICKHOUSE_DDL_FILE],
+      ...(warnings.length > 0 ? { warnings } : {}),
     };
   },
 };