@intentius/chant-lexicon-sql 0.117.0 → 0.118.0

package/src/files/postgres.ts

@@ -0,0 +1,371 @@
+/**
+ * A file of Postgres DDL as the objects it declares: each `CREATE` (and each
+ * `GRANT`, `REVOKE` and `ALTER DEFAULT PRIVILEGES`) is one object, and the
+ * statements that only finish one are folded into it:
+ *
+ * - `COMMENT ON` joins the object it comments on, wherever in the file it is;
+ * - `ALTER TABLE ... ENABLE | DISABLE | FORCE ROW LEVEL SECURITY` joins its table;
+ * - `ALTER TABLE <t> ADD [CONSTRAINT <c>] FOREIGN KEY | UNIQUE | PRIMARY KEY |
+ *   CHECK | EXCLUDE ...`, the form `pg_dump` and ORMs print constraints in,
+ *   becomes a table constraint in `<t>`'s `CREATE TABLE`.
+ *
+ * `BEGIN` and `COMMIT` are skipped. Any other statement, and any statement
+ * the parser or its tag refuses, is a problem naming it: a statement left
+ * out would be a schema change nobody declared.
+ *
+ * With `schema`, the names the DDL leaves unqualified are qualified with it,
+ * as an ORM's connection would: an object's own name, an index's, trigger's
+ * or policy's table, a foreign key's table, a comment's target, and a column
+ * type naming an enum or domain the file creates.
+ */
+
+import type { Declarable } from "@intentius/chant/declarable";
+import { isTrivia, tokenizeText, type Token } from "../postgres/tokens";
+import { identValue, parseStatements, type NameNode, type StatementNode } from "../postgres/parser";
+import * as tags from "../postgres/entities";
+import { POSTGRES_ENTITY_TYPES, type PostgresEntityType } from "../postgres/entity-types";
+import { POSTGRES_TAG_OF } from "../postgres/import/generator";
+import type { ImportedPgObject } from "../postgres/import/ir";
+import { firstLine, quoteBare, splitTokens, tokensText, type ReadOptions } from "./common";
+
+type Tag = (strings: TemplateStringsArray, ...values: unknown[]) => Declarable;
+
+/** The tag that declares an entity type. */
+export function postgresTag(type: string): Tag | undefined {
+  const name = POSTGRES_TAG_OF[type];
+  return name ? ((tags as unknown as Record<string, Tag>)[name] ?? undefined) : undefined;
+}
+
+/** A tagged-template call's strings from template-literal text: `raw` as written, as the tag reads it. */
+export function templateStrings(parts: readonly string[]): TemplateStringsArray {
+  return Object.assign([...parts], { raw: [...parts] }) as unknown as TemplateStringsArray;
+}
+
+const isName = (t: Token | undefined): boolean => t !== undefined && (t.kind === "ident" || t.kind === "qident");
+const word = (t: Token | undefined, ...words: string[]): boolean => t !== undefined && t.kind === "ident" && words.includes(t.text.toUpperCase());
+
+/** Text that sits inside a template literal: a backquote and `${` escaped. */
+const templateSafe = (s: string) => s.replace(/\\(?=`|\$\{)/g, "\\\\").replace(/`/g, "\\`").replace(/\$\{/g, "\\${");
+
+interface PgFileObject {
+  node: StatementNode;
+  /** The CREATE's tokens, then each statement folded into it. */
+  statements: Token[][];
+  /** Table constraints folded in from ALTER TABLE ... ADD, as SQL text. */
+  constraints: string[];
+}
+
+/** Transaction control, which declares nothing. */
+function isTransactionControl(sig: readonly Token[]): boolean {
+  const words = sig.map((t) => (t.kind === "ident" ? t.text.toUpperCase() : ""));
+  const [first, ...rest] = words;
+  if (first === "BEGIN" || first === "COMMIT" || first === "END") return rest.every((w) => w === "TRANSACTION" || w === "WORK");
+  return first === "START" && rest.length === 1 && rest[0] === "TRANSACTION";
+}
+
+/**
+ * The objects a file of Postgres DDL declares, each with its statements
+ * joined as one declaration's DDL. Problems are collected, not thrown, so a
+ * file is reported whole.
+ */
+export function readPostgres(ddl: string, options: ReadOptions, problems: string[]): ImportedPgObject[] {
+  const parsed: Array<{ tokens: Token[]; node: StatementNode }> = [];
+  const alters: Array<{ table: string[]; constraint: string; sql: string }> = [];
+  for (const stmt of splitTokens(tokenizeText(ddl, 0), isTrivia)) {
+    const sig = stmt.filter((t) => !isTrivia(t));
+    const sql = tokensText(stmt);
+    if (isTransactionControl(sig)) continue;
+    if (word(sig[0], "ALTER") && word(sig[1], "TABLE")) {
+      const alter = readAlterAdd(stmt, sig);
+      if (alter) {
+        alters.push({ ...alter, sql });
+        continue;
+      }
+    }
+    if (!word(sig[0], "CREATE", "COMMENT", "GRANT", "REVOKE", "ALTER")) {
+      problems.push(`not a statement that declares an object: ${firstLine(sql)}`);
+      continue;
+    }
+    try {
+      const tokens = tokenizeText(sql, 0);
+      for (const node of parseStatements(tokens)) parsed.push({ tokens, node });
+    } catch (e) {
+      const what = word(sig[0], "ALTER") && word(sig[1], "TABLE") ? "an ALTER TABLE other than ADD of a table constraint or ROW LEVEL SECURITY" : `does not parse (${e instanceof Error ? e.message : String(e)})`;
+      problems.push(`${what}: ${firstLine(sql)}`);
+    }
+  }
+
+  const nameOf = (tokens: Token[], n: NameNode): string[] =>
+    n.pieces.length > 0 && n.pieces.every((p) => p !== "") ? n.pieces : tokens.slice(n.span.from, n.span.to).filter((t) => isName(t)).map((t) => identValue(t));
+  const qualify = (pieces: readonly string[]): { schema?: string; name: string } => {
+    const name = pieces[pieces.length - 1] ?? "";
+    const schema = pieces.length >= 2 ? pieces[pieces.length - 2] : options.schema;
+    return schema ? { schema, name } : { name };
+  };
+  const key = (q: { schema?: string; name: string }) => `${q.schema ?? ""}.${q.name}`;
+
+  // The types the file creates, so a column of one is qualified with the rest.
+  const fileTypes = new Set<string>();
+  for (const { tokens, node } of parsed) {
+    if (node.statement === "enum" || node.statement === "domain") {
+      const pieces = nameOf(tokens, node.name);
+      if (pieces.length === 1) fileTypes.add(pieces[0]!);
+    }
+  }
+
+  const objects: PgFileObject[] = [];
+  /** Objects by namespace and name: `rel` (anything in a schema), `index`, `schema`, `extension`, `role`, and `trigger`/`policy` with their table. */
+  const owners = new Map<string, PgFileObject>();
+  const followOns: Array<{ tokens: Token[]; node: StatementNode; inserts: Set<number> }> = [];
+  for (const { tokens, node } of parsed) {
+    const inserts = new Set<number>();
+    /** Qualifies an unqualified name in the statement, when `schema` is set. */
+    const mark = (n: NameNode | undefined) => {
+      if (!n || !options.schema) return;
+      const names = tokens.slice(n.span.from, n.span.to).filter((t) => !isTrivia(t));
+      const [only] = names;
+      if (only && names.length === 1 && isName(only)) inserts.add(tokens.indexOf(only));
+    };
+    switch (node.statement) {
+      case "comment":
+        if (node.objectType !== "SCHEMA" && node.objectType !== "EXTENSION" && node.objectType !== "ROLE" && node.objectType !== "TRIGGER" && node.objectType !== "POLICY" && node.objectType !== "CONSTRAINT" && node.objectType !== "COLUMN") mark(node.target);
+        if (node.objectType === "COLUMN" && nameOf(tokens, node.target).length === 2 && options.schema) {
+          const first = tokens.slice(node.target.span.from, node.target.span.to).find((t) => !isTrivia(t));
+          if (first) inserts.add(tokens.indexOf(first));
+        }
+        mark(node.on);
+        followOns.push({ tokens, node, inserts });
+        continue;
+      case "rowSecurity":
+        mark(node.table);
+        followOns.push({ tokens, node, inserts });
+        continue;
+      case "index":
+      case "trigger":
+      case "policy":
+        mark(node.table);
+        break;
+      case "table":
+        for (const c of [...node.columns.flatMap((col) => col.constraints), ...node.constraints]) mark(c.references?.table);
+        if (options.schema) {
+          for (const col of node.columns) {
+            if (!col.type) continue;
+            const type = tokens.slice(col.type.from, col.type.to).filter((t) => !isTrivia(t));
+            const first = type[0];
+            if (first && isName(first) && fileTypes.has(identValue(first)) && !(type[1]?.kind === "punct" && type[1].text === ".")) inserts.add(tokens.indexOf(first));
+          }
+        }
+        break;
+      default:
+        break;
+    }
+    if (node.statement === "schema" && !node.name) {
+      problems.push(`a CREATE SCHEMA with no name: ${firstLine(tokensText(tokens))}`);
+      continue;
+    }
+    if (node.statement === "index" && !node.name) {
+      problems.push(`an index with no name (its declaration needs one): ${firstLine(tokensText(tokens))}`);
+      continue;
+    }
+    const ownName = !["schema", "extension", "index", "trigger", "policy", "role", "grant"].includes(node.statement);
+    if (ownName && "name" in node) mark(node.name);
+    const object: PgFileObject = { node, statements: [rewrite(tokens, inserts, options.schema)], constraints: [] };
+    objects.push(object);
+    const k = ownerKey(node, tokens, nameOf, qualify);
+    if (k) owners.set(k, object);
+  }
+
+  for (const f of followOns) {
+    const k = followOnOwner(f.node, f.tokens, nameOf, qualify);
+    const owner = k ? owners.get(k) : undefined;
+    if (!owner) {
+      const what = f.node.statement === "comment" ? `COMMENT ON ${f.node.objectType}` : "ALTER TABLE ... ROW LEVEL SECURITY";
+      problems.push(`${what} for an object the file does not create: ${firstLine(tokensText(f.tokens))}`);
+      continue;
+    }
+    owner.statements.push(rewrite(f.tokens, f.inserts, options.schema));
+  }
+
+  for (const alter of alters) {
+    const table = owners.get(`rel ${key(qualify(alter.table))}`);
+    if (!table || table.node.statement !== "table") {
+      problems.push(`ALTER TABLE of a table the file does not create: ${firstLine(alter.sql)}`);
+      continue;
+    }
+    table.constraints.push(qualifyConstraint(alter.constraint, options.schema));
+  }
+
+  const out: ImportedPgObject[] = [];
+  for (const o of objects) {
+    const [create = [], ...rest] = o.statements;
+    const ddlText = [o.constraints.length > 0 ? addConstraints(create, o.constraints) : tokensText(create), ...rest.map((c) => tokensText(c))].join(";\n");
+    const identity = identify(ddlText);
+    if (typeof identity === "string") {
+      problems.push(`${identity}: ${firstLine(ddlText)}`);
+      continue;
+    }
+    out.push({ ...identity, ddl: ddlText });
+  }
+  return out;
+}
+
+/** An object's entity type, schema and name, read by declaring it with its tag; a string is the tag's refusal. */
+function identify(ddl: string): { type: PostgresEntityType; schema?: string; name: string } | string {
+  let tokens: Token[];
+  let node: StatementNode | undefined;
+  try {
+    tokens = tokenizeText(ddl, 0);
+    [node] = parseStatements(tokens);
+  } catch (e) {
+    return `does not parse (${e instanceof Error ? e.message : String(e)})`;
+  }
+  const type = node ? entityTypeOf(node) : undefined;
+  const tag = type ? postgresTag(type) : undefined;
+  if (!type || !tag) return "not a statement that declares an object";
+  try {
+    const entity = tag(templateStrings([templateSafe(ddl)]));
+    const p = (entity as unknown as { props: { schema?: string; name: string } }).props;
+    return { type: entity.entityType as PostgresEntityType, ...(p.schema ? { schema: p.schema } : {}), name: p.name };
+  } catch (e) {
+    return e instanceof Error ? e.message : String(e);
+  }
+}
+
+/** The entity type a statement declares. */
+function entityTypeOf(node: StatementNode): PostgresEntityType | undefined {
+  switch (node.statement) {
+    case "view":
+      return node.materialized ? POSTGRES_ENTITY_TYPES.materializedView : POSTGRES_ENTITY_TYPES.view;
+    case "grant":
+      return node.defaults ? POSTGRES_ENTITY_TYPES.defaultPrivileges : POSTGRES_ENTITY_TYPES.grant;
+    case "comment":
+    case "rowSecurity":
+      return undefined;
+    default:
+      return (POSTGRES_ENTITY_TYPES as Record<string, PostgresEntityType>)[node.statement];
+  }
+}
+
+type NameOf = (tokens: Token[], n: NameNode) => string[];
+type Qualify = (pieces: readonly string[]) => { schema?: string; name: string };
+const qkey = (q: { schema?: string; name: string }) => `${q.schema ?? ""}.${q.name}`;
+
+/** The key a created object is found under by the statements folded into it. */
+function ownerKey(node: StatementNode, tokens: Token[], nameOf: NameOf, qualify: Qualify): string | undefined {
+  switch (node.statement) {
+    case "schema":
+      return node.name ? `schema ${nameOf(tokens, node.name).at(-1) ?? ""}` : undefined;
+    case "extension":
+      return `extension ${nameOf(tokens, node.name).at(-1) ?? ""}`;
+    case "role":
+      return `role ${nameOf(tokens, node.name).at(-1) ?? ""}`;
+    case "index": {
+      // An index is in its table's schema.
+      const table = nameOf(tokens, node.table);
+      return node.name ? `index ${qkey(qualify([...table.slice(0, -1), nameOf(tokens, node.name).at(-1) ?? ""]))}` : undefined;
+    }
+    case "trigger":
+    case "policy":
+      return `${node.statement} ${nameOf(tokens, node.name).at(-1) ?? ""} ON ${qkey(qualify(nameOf(tokens, node.table)))}`;
+    case "grant":
+    case "comment":
+    case "rowSecurity":
+      return undefined;
+    default:
+      return `rel ${qkey(qualify(nameOf(tokens, node.name)))}`;
+  }
+}
+
+/** The key of the object a COMMENT ON or ALTER TABLE ... ROW LEVEL SECURITY finishes. */
+function followOnOwner(node: StatementNode, tokens: Token[], nameOf: NameOf, qualify: Qualify): string | undefined {
+  if (node.statement === "rowSecurity") return `rel ${qkey(qualify(nameOf(tokens, node.table)))}`;
+  if (node.statement !== "comment") return undefined;
+  const target = nameOf(tokens, node.target);
+  switch (node.objectType) {
+    case "SCHEMA":
+      return `schema ${target.at(-1) ?? ""}`;
+    case "EXTENSION":
+      return `extension ${target.at(-1) ?? ""}`;
+    case "ROLE":
+      return `role ${target.at(-1) ?? ""}`;
+    case "INDEX":
+      return `index ${qkey(qualify(target))}`;
+    case "COLUMN":
+      return `rel ${qkey(qualify(target.slice(0, -1)))}`;
+    case "TRIGGER":
+    case "POLICY":
+      return node.on ? `${node.objectType.toLowerCase()} ${target.at(-1) ?? ""} ON ${qkey(qualify(nameOf(tokens, node.on)))}` : undefined;
+    case "CONSTRAINT":
+    case "DOMAIN CONSTRAINT":
+      return node.on ? `rel ${qkey(qualify(nameOf(tokens, node.on)))}` : undefined;
+    default:
+      return `rel ${qkey(qualify(target))}`;
+  }
+}
+
+/** The statement's tokens with `schema.` inserted before each marked one. */
+function rewrite(tokens: readonly Token[], inserts: ReadonlySet<number>, schema: string | undefined): Token[] {
+  if (!schema || inserts.size === 0) return [...tokens];
+  const out: Token[] = [];
+  tokens.forEach((t, i) => {
+    if (inserts.has(i)) out.push({ kind: "ident", text: `${quoteBare(schema)}.`, part: 0, start: 0, end: 0 });
+    out.push(t);
+  });
+  return out;
+}
+
+/** `ALTER TABLE [IF EXISTS] [ONLY] <name> ADD <table constraint>`: the table's name pieces and the constraint's text. */
+function readAlterAdd(stmt: readonly Token[], sig: readonly Token[]): { table: string[]; constraint: string } | undefined {
+  let i = 2;
+  if (word(sig[i], "IF") && word(sig[i + 1], "EXISTS")) i += 2;
+  if (word(sig[i], "ONLY")) i++;
+  const table: string[] = [];
+  for (let t = sig[i]; t && isName(t); t = sig[i]) {
+    table.push(identValue(t));
+    i++;
+    const dot = sig[i];
+    if (dot?.kind === "punct" && dot.text === ".") i++;
+    else break;
+  }
+  if (table.length === 0 || !word(sig[i], "ADD")) return undefined;
+  i++;
+  if (!word(sig[i], "CONSTRAINT", "FOREIGN", "UNIQUE", "PRIMARY", "CHECK", "EXCLUDE")) return undefined;
+  // One constraint: a comma at depth 0 starts another ALTER action, which is not read.
+  let depth = 0;
+  for (const t of sig.slice(i)) {
+    if (t.kind === "punct" && t.text === "(") depth++;
+    else if (t.kind === "punct" && t.text === ")") depth--;
+    else if (t.kind === "punct" && t.text === "," && depth === 0) return undefined;
+  }
+  const start = sig[i];
+  return start ? { table, constraint: tokensText(stmt.slice(stmt.indexOf(start))) } : undefined;
+}
+
+/** Qualifies the table a folded FOREIGN KEY references, when it is unqualified. */
+function qualifyConstraint(constraint: string, schema: string | undefined): string {
+  if (!schema) return constraint;
+  const tokens = tokenizeText(constraint, 0);
+  const sig = tokens.map((t, i) => ({ t, i })).filter((x) => !isTrivia(x.t));
+  const inserts = new Set<number>();
+  sig.forEach((x, k) => {
+    const target = sig[k + 1];
+    const dot = sig[k + 2]?.t;
+    if (word(x.t, "REFERENCES") && target && isName(target.t) && !(dot?.kind === "punct" && dot.text === ".")) inserts.add(target.i);
+  });
+  return tokensText(rewrite(tokens, inserts, schema));
+}
+
+/** A CREATE TABLE with table constraints added at the end of its column list (the `)` matching the first `(`). */
+function addConstraints(create: readonly Token[], constraints: readonly string[]): string {
+  let depth = 0;
+  const close = create.findIndex((t) => {
+    if (t.kind !== "punct") return false;
+    if (t.text === "(") depth++;
+    else if (t.text === ")") return --depth === 0;
+    return false;
+  });
+  if (close < 0) return tokensText(create);
+  const before = tokensText(create.slice(0, close)).replace(/\s+$/, "");
+  const indent = /\n([ \t]+)\S[^\n]*$/.exec(before)?.[1] ?? "  ";
+  return `${before}${constraints.map((c) => `,\n${indent}${c}`).join("")}\n${tokensText(create.slice(close))}`;
+}

package/src/generated/lexicon-sql.json

@@ -19,6 +19,36 @@
     "kind": "resource",
     "lexicon": "sql"
   },
+  "Dictionary": {
+    "resourceType": "ClickHouse::Dictionary",
+    "kind": "resource",
+    "lexicon": "sql"
+  },
+  "Function": {
+    "resourceType": "ClickHouse::Function",
+    "kind": "resource",
+    "lexicon": "sql"
+  },
+  "User": {
+    "resourceType": "ClickHouse::User",
+    "kind": "resource",
+    "lexicon": "sql"
+  },
+  "Role": {
+    "resourceType": "ClickHouse::Role",
+    "kind": "resource",
+    "lexicon": "sql"
+  },
+  "RowPolicy": {
+    "resourceType": "ClickHouse::RowPolicy",
+    "kind": "resource",
+    "lexicon": "sql"
+  },
+  "Grant": {
+    "resourceType": "ClickHouse::Grant",
+    "kind": "resource",
+    "lexicon": "sql"
+  },
   "PostgresSchema": {
     "resourceType": "Postgres::Schema",
     "kind": "resource",

package/src/import-parser.ts

@@ -0,0 +1,61 @@
+/**
+ * `chant import schema.sql`: a file of DDL into the import IR, in either
+ * dialect. `--parser-option dialect=postgres` (or `clickhouse`) names it;
+ * without one it is read off the statements: an `ENGINE =` clause, a
+ * `CREATE DATABASE`, a `CREATE DICTIONARY` or a lambda `CREATE FUNCTION f AS
+ * (x) -> ...` is ClickHouse, a statement
+ * only Postgres has (`CREATE SCHEMA`, `CREATE INDEX`, `COMMENT ON`, `ALTER
+ * TABLE`, `GRANT`, ...) or a `CREATE TABLE` with no engine is Postgres, and
+ * anything else is ClickHouse.
+ *
+ * A statement that declares nothing, or does not parse, is an error naming
+ * it (`./sql-files.ts`); none is left out.
+ */
+
+import type { ParserOptions, TemplateIR, TemplateParser } from "@intentius/chant/import/parser";
+import { SQL_DIALECTS, type SqlDialect } from "./dialects";
+import { readClickHouse } from "./files/clickhouse";
+import { readPostgres } from "./files/postgres";
+import { SqlFileError } from "./files/common";
+import { objectsToIR as chObjectsToIR } from "./clickhouse/import/ir";
+import { objectsToIR as pgObjectsToIR } from "./postgres/import/ir";
+
+const CLICKHOUSE_ONLY = [
+  /\bENGINE\s*=/i,
+  /^\s*CREATE\s+(?:DATABASE|(?:OR\s+REPLACE\s+)?DICTIONARY)\b/im,
+  // A SQL function: CREATE FUNCTION f AS (x) -> ...
+  /^\s*CREATE\s+(?:OR\s+REPLACE\s+)?FUNCTION\s+[^\s(]+(?:\s+ON\s+CLUSTER\s+\S+)?\s+AS\s*\(/im,
+];
+const POSTGRES_ONLY = [
+  /^\s*CREATE\s+(?:OR\s+REPLACE\s+)?(?:SCHEMA|EXTENSION|TYPE|DOMAIN|SEQUENCE|(?:UNIQUE\s+)?INDEX|(?:CONSTRAINT\s+)?TRIGGER|POLICY|ROLE|FUNCTION|PROCEDURE|(?:UNLOGGED|TEMP|TEMPORARY)\s+TABLE)\b/im,
+  /^\s*(?:COMMENT\s+ON|ALTER\s+TABLE|ALTER\s+DEFAULT\s+PRIVILEGES|GRANT|REVOKE|BEGIN)\b/im,
+  /^\s*CREATE\s+TABLE\b/im,
+];
+
+/** The dialect a file of DDL is written in, read off its statements. */
+export function guessDialect(content: string): SqlDialect {
+  const text = content.replace(/--[^\n]*/g, "");
+  if (CLICKHOUSE_ONLY.some((re) => re.test(text))) return "clickhouse";
+  if (POSTGRES_ONLY.some((re) => re.test(text))) return "postgres";
+  return "clickhouse";
+}
+
+/** The `dialect` parser option, checked. */
+export function dialectOption(options: ParserOptions | undefined): SqlDialect | undefined {
+  const d = options?.dialect;
+  if (d === undefined) return undefined;
+  if (typeof d !== "string" || !(SQL_DIALECTS as readonly string[]).includes(d)) throw new Error(`--parser-option dialect: expected ${SQL_DIALECTS.join(" or ")}, got ${String(d)}`);
+  return d as SqlDialect;
+}
+
+export class SqlFileParser implements TemplateParser {
+  constructor(private readonly dialect?: SqlDialect) {}
+
+  parse(content: string): TemplateIR {
+    const dialect = this.dialect ?? guessDialect(content);
+    const problems: string[] = [];
+    const ir = dialect === "postgres" ? pgObjectsToIR(readPostgres(content, { origin: "the DDL" }, problems)) : chObjectsToIR(readClickHouse(content, { origin: "the DDL" }, problems));
+    if (problems.length > 0) throw new SqlFileError(`the ${dialect === "postgres" ? "Postgres" : "ClickHouse"} DDL`, problems);
+    return ir;
+  }
+}

package/src/index.ts

@@ -12,13 +12,13 @@ export { sqlConfigSchema, type SqlConfig } from "./config";
 // them as intrinsics (check-lexicon resolves a registered intrinsic against the
 // root's exports). Declarations import them from the dialect subpath,
 // `@intentius/chant-lexicon-sql/clickhouse`.
-export { database, table, view, literal } from "./clickhouse/entities";
+export { database, table, view, dictionary, user, role, policy, grant, literal } from "./clickhouse/entities";
 
 // The Postgres tags whose names ClickHouse does not export, for the same reason.
 // A tag folds as the function its file imports, so `table` imported from
 // `@intentius/chant-lexicon-sql/postgres` is Postgres's. Declarations import
 // them from the dialect subpath.
-export { schema, index, sequence, type, domain, extension, func, procedure, trigger, policy, role, grant } from "./postgres/entities";
+export { schema, index, sequence, type, domain, extension, func, procedure, trigger } from "./postgres/entities";
 
 // A schema change's statements between two builds, rendered offline for a
 // migration file (#3644); also at `@intentius/chant-lexicon-sql/migration-statements`.
@@ -34,3 +34,16 @@ export {
   type StatementStep,
   type StepObject,
 } from "./migration-statements";
+
+// Plain `.sql` files as a schema source (#3646): DDL read into the declarations
+// the tagged templates make; also at `@intentius/chant-lexicon-sql/sql-files`.
+export {
+  readSqlFile,
+  sqlFileDeclarations,
+  sqlFileEntities,
+  sqlFileOf,
+  SqlFileError,
+  type SqlFileObject,
+  type SqlFileOptions,
+  type SqlFileSource,
+} from "./sql-files";

package/src/lint/rules/sqlch001.ts

@@ -1,12 +1,18 @@
 import type { LintContext, LintDiagnostic, LintRule } from "@intentius/chant/lint/rule";
 import { parseCreate } from "../../clickhouse/parser";
+import { parseAccess } from "../../clickhouse/access";
 import { SqlSyntaxError } from "../../clickhouse/tokens";
 import { findTemplates, templatePosition, tokensOf } from "./templates";
 
-const STATEMENT: Record<string, string> = { database: "CREATE DATABASE", table: "CREATE TABLE", view: "CREATE VIEW" };
+const ACCESS_TAGS = new Set(["user", "role", "policy", "grant"]);
+const ACCESS_STATEMENT: Record<string, string> = { user: "CREATE USER", role: "CREATE ROLE", rowPolicy: "CREATE ROW POLICY", grant: "GRANT" };
+
+const STATEMENT: Record<string, string> = { database: "CREATE DATABASE", table: "CREATE TABLE", view: "CREATE VIEW", dictionary: "CREATE DICTIONARY", function: "CREATE FUNCTION" };
+/** The tag a statement belongs in: `func` for a function, the statement's own name otherwise. */
+const tagOf = (statement: string): string => (statement === "function" ? "func" : statement);
 
 /**
- * SQLCH001: the DDL in a `database`, `table` or `view` template does not parse, or holds another statement than its tag.
+ * SQLCH001: the DDL in a ClickHouse template (`database`, `table`, `view`, `dictionary`, `func`, `user`, `role`, `policy`, `grant`) does not parse, or holds another statement than its tag.
  *
  * The same parse runs when the build calls the tag, and fails the build there.
  * This rule reports it in the editor and in `chant lint`, at the token, before
@@ -27,12 +33,28 @@ export const sqlch001: LintRule = {
       let tokens;
       try {
         tokens = tokensOf(found);
+        if (ACCESS_TAGS.has(found.tag)) {
+          const node = parseAccess(tokens);
+          const tag = node.statement === "rowPolicy" ? "policy" : node.statement;
+          if (tag !== found.tag || node.revoke) {
+            out.push({
+              ruleId: "SQLCH001",
+              severity: "error",
+              message: node.revoke
+                ? `${found.tag}\`...\` holds a REVOKE; declare the grants a grantee keeps, and a plan revokes the rest`
+                : `${found.tag}\`...\` holds a ${ACCESS_STATEMENT[node.statement]}; use the ${tag} tag`,
+              file: context.filePath,
+              ...templatePosition(source, found, 0, 0),
+            });
+          }
+          continue;
+        }
         const node = parseCreate(tokens);
-        if (node.statement !== found.tag) {
+        if (tagOf(node.statement) !== found.tag) {
           out.push({
             ruleId: "SQLCH001",
             severity: "error",
-            message: `${found.tag}\`...\` holds a ${STATEMENT[node.statement]}; use the ${node.statement} tag`,
+            message: `${found.tag}\`...\` holds a ${STATEMENT[node.statement]}; use the ${tagOf(node.statement)} tag`,
             file: context.filePath,
             ...templatePosition(source, found, 0, 0),
           });

package/src/lint/rules/sqlch002.ts

@@ -41,7 +41,8 @@ function keyColumns(tokens: Token[], span: Span): Array<{ name: string; token: T
 }
 
 function allowsNullableKey(tokens: Token[], node: CreateNode): boolean {
-  if (node.statement === "database" || !node.settings) return false;
+  if (node.statement !== "table" && node.statement !== "view") return false;
+  if (!node.settings) return false;
   const s = node.settings.find((x) => x.key === "allow_nullable_key");
   if (!s) return false;
   const v = tokens.slice(s.value.from, s.value.to).map((t) => t.text).join("").trim().toLowerCase().replace(/'/g, "");
@@ -74,7 +75,7 @@ export const sqlch002: LintRule = {
         if (err instanceof SqlSyntaxError) continue; // SQLCH001 reports it
         throw err;
       }
-      if (node.statement === "database" || allowsNullableKey(tokens, node)) continue;
+      if ((node.statement !== "table" && node.statement !== "view") || allowsNullableKey(tokens, node)) continue;
       const nullable = new Set(node.columns.filter((c) => c.name && isNullable(tokens, c)).map((c) => c.name));
       if (nullable.size === 0) continue;
       const reported = new Set<string>();

package/src/lint/rules/templates.ts

@@ -13,7 +13,7 @@ import { findSqlTemplates, type FoundTemplate as SqlFoundTemplate, type Template
 
 export { templatePosition, tokenPosition } from "../../core/find-templates";
 
-export type SqlTag = "database" | "table" | "view";
+export type SqlTag = "database" | "table" | "view" | "dictionary" | "func" | "user" | "role" | "policy" | "grant";
 
 /**
  * The ClickHouse tags, by the modules they are imported from. The package
@@ -22,7 +22,7 @@ export type SqlTag = "database" | "table" | "view";
 export const CLICKHOUSE_TEMPLATE_SOURCE: TemplateSource<SqlTag> = {
   dialect: "clickhouse",
   modules: ["@intentius/chant-lexicon-sql", "@intentius/chant-lexicon-sql/clickhouse"],
-  tags: ["database", "table", "view"],
+  tags: ["database", "table", "view", "dictionary", "func", "user", "role", "policy", "grant"],
 };
 
 export type FoundTemplate = SqlFoundTemplate<SqlTag>;

package/src/lsp/template.ts

@@ -143,13 +143,13 @@ export function declarations(content: string, fileName = "file.ts"): Declared[]
       const tokens = tokenize(found.parts);
       const node = parseCreate(tokens);
       const columns =
-        node.statement === "database"
+        node.statement === "database" || node.statement === "function"
           ? []
-          : node.columns.map((c) => ({
+          : (node.statement === "dictionary" ? node.attributes : node.columns).map((c) => ({
               name: c.name || spanText(found, tokens, c.nameSpan, source) || "",
               ...(spanText(found, tokens, c.type, source) ? { type: spanText(found, tokens, c.type, source)! } : {}),
             }));
-      const engine = node.engine?.name || spanText(found, tokens, node.engine?.nameSpan, source);
+      const engine = node.statement === "dictionary" || node.statement === "function" ? undefined : node.engine?.name || spanText(found, tokens, node.engine?.nameSpan, source);
       out.push({
         name: decl.name.text,
         tag: found.tag,

package/src/mcp/index.ts

@@ -10,7 +10,7 @@ import type { McpResourceContribution, McpToolContribution } from "@intentius/ch
 import { createCatalogResource, createDiffTool } from "@intentius/chant/lexicon-plugin-helpers";
 import { sqlSerializer } from "../serializer";
 import { catalogIndex } from "../lsp/catalog";
-import { database, table, view } from "../clickhouse/entities";
+import { database, dictionary, func, grant, policy, role, table, user, view } from "../clickhouse/entities";
 import { SqlSyntaxError } from "../clickhouse/tokens";
 import { existsSync, readFileSync } from "fs";
 import { diffSchemas } from "../clickhouse/plan/diff";
@@ -122,12 +122,12 @@ const searchTool: McpToolContribution = {
 const parseTool: McpToolContribution = {
   name: "parse-ddl",
   description:
-    "Parse one CREATE statement the way its tag does (dialect clickhouse, the default: table, view, database; dialect postgres: schema, table, index, view, sequence, type, domain, extension), and return the entity it builds (columns, engine, keys, settings, and for a view its lineage). A statement that does not parse comes back as the error with its line and column, the same one SQLCH001 (ClickHouse) or SQLPG001 (Postgres) reports. The statement is plain DDL: there are no interpolations.",
+    "Parse one CREATE statement the way its tag does (dialect clickhouse, the default: table, view, database, dictionary, func, user, role, policy, grant; dialect postgres: schema, table, index, view, sequence, type, domain, extension), and return the entity it builds (columns, engine, keys, settings, and for a view its lineage). A statement that does not parse comes back as the error with its line and column, the same one SQLCH001 (ClickHouse) or SQLPG001 (Postgres) reports. The statement is plain DDL: there are no interpolations.",
   inputSchema: {
     type: "object",
     properties: {
       dialect: dialectProp,
-      tag: { type: "string", enum: [...new Set(["table", "view", "database", ...POSTGRES_TAGS])], description: "The tag the statement belongs in (ClickHouse: table, view, database; Postgres: " + POSTGRES_TAGS.join(", ") + ")" },
+      tag: { type: "string", enum: [...new Set(["table", "view", "database", "dictionary", "func", "user", "role", "policy", "grant", ...POSTGRES_TAGS])], description: "The tag the statement belongs in (ClickHouse: table, view, database, dictionary, func, user, role, policy, grant; Postgres: " + POSTGRES_TAGS.join(", ") + ")" },
       ddl: { type: "string", description: "The CREATE statement" },
     },
     required: ["tag", "ddl"],
@@ -136,8 +136,8 @@ const parseTool: McpToolContribution = {
     const ddl = String(params.ddl ?? "");
     const tag = String(params.tag);
     if (asDialect(params.dialect) === "postgres") return postgresParse(tag, ddl, position);
-    const build = tag === "table" ? table : tag === "view" ? view : tag === "database" ? database : undefined;
-    if (!build) throw new Error("tag must be table, view or database");
+    const build = tag === "table" ? table : tag === "view" ? view : tag === "database" ? database : tag === "dictionary" ? dictionary : tag === "func" ? func : tag === "user" ? user : tag === "role" ? role : tag === "policy" ? policy : tag === "grant" ? grant : undefined;
+    if (!build) throw new Error("tag must be table, view, database, dictionary, func, user, role, policy or grant");
     const strings = Object.assign([ddl], { raw: [ddl] }) as unknown as TemplateStringsArray;
     try {
       const entity = build(strings);
@@ -276,7 +276,7 @@ export function sqlMcpResources(): McpResourceContribution[] {
     createCatalogResource(
       new URL("../plugin.ts", import.meta.url).href,
       "SQL Entity Catalog",
-      "The entity kinds the sql lexicon declares (database, table, view, materialized view)",
+      "The entity kinds the sql lexicon declares (database, table, view, materialized view, dictionary, function, user, role, row policy, grant)",
       "lexicon-sql.json",
       "sql",
     ),