@intentius/chant-lexicon-sql 0.0.0-stage → 0.112.1

package/src/core/ownership.ts

@@ -0,0 +1,133 @@
+/**
+ * chant's ownership marker as a trailer on an object's own comment, in every
+ * dialect that can comment on its objects (#3208).
+ *
+ * The marker never replaces the user's comment, which is a declared property.
+ * It is appended as a bracketed trailer,
+ *
+ *     Raw events [chant managed-by=chant stack=shop env=prod]
+ *
+ * and taken off again wherever chant reads a definition, so planning, the deep
+ * diff and import compare and write the declared comment alone. Someone who
+ * edits the comment by hand and drops the trailer makes the object read as
+ * foreign, which is the safe direction: it is never pruned, and the next apply
+ * of its declaration stamps it again.
+ *
+ * Values are percent-encoded outside `[A-Za-z0-9._-]`, so the trailer never
+ * holds a quote, a backslash, a space or a bracket of its own.
+ *
+ * Where the comment lives in a statement, and so how a trailer comes off a
+ * printed `CREATE`, is the dialect's grammar and stays with the dialect.
+ */
+
+import { OWNERSHIP_MANAGED_BY_VALUE, type ChannelKeys, type OwnershipMarker } from "@intentius/chant/ownership";
+
+/** The keys inside the trailer. */
+export const COMMENT_OWNERSHIP_KEYS: ChannelKeys = {
+  managedBy: "managed-by",
+  stack: "stack",
+  env: "env",
+};
+
+/**
+ * The trailer key on the objects that hold effect receipts. Like a
+ * migration's working objects, they are chant's own bookkeeping and never
+ * part of a declared schema.
+ */
+export const RECEIPTS_TRAILER_KEY = "receipts";
+
+const VALUE = "[A-Za-z0-9._%-]*";
+/** One `key=value` pair of a trailer, as a regular expression source. */
+export const TRAILER_PAIR = `[A-Za-z0-9._-]+=${VALUE}`;
+/** The trailer at the end of a comment's text. */
+const TRAILER = new RegExp(`(?:^|\\s+)\\[chant((?: ${TRAILER_PAIR})+)\\]$`);
+
+const encode = (v: string) => v.replace(/[^A-Za-z0-9._-]/g, (c) => [...new TextEncoder().encode(c)].map((b) => `%${b.toString(16).toUpperCase().padStart(2, "0")}`).join(""));
+const decode = (v: string) => {
+  try {
+    return decodeURIComponent(v);
+  } catch {
+    return v;
+  }
+};
+
+/** What a trailer says: the managed-by value and, when stamped, the stack and env. */
+export interface CommentMarker {
+  managedBy: string;
+  stack?: string;
+  env?: string;
+}
+
+/**
+ * The trailer for a marker: `[chant managed-by=chant stack=<stack> env=<env>]`.
+ * `extra` appends more pairs after the marker's own; a migration Op uses them
+ * to say which migration an object it made belongs to.
+ * Keys are `[A-Za-z0-9._-]+`; values are encoded like the rest.
+ */
+export function markerTrailer(marker: OwnershipMarker | undefined, extra: Readonly<Record<string, string>> = {}): string {
+  const k = COMMENT_OWNERSHIP_KEYS;
+  const pairs = [`${k.managedBy}=${OWNERSHIP_MANAGED_BY_VALUE}`];
+  if (marker?.stack) pairs.push(`${k.stack}=${encode(marker.stack)}`);
+  if (marker?.env) pairs.push(`${k.env}=${encode(marker.env)}`);
+  for (const [key, value] of Object.entries(extra)) {
+    if (!/^[A-Za-z0-9._-]+$/.test(key)) throw new Error(`ownership trailer key ${JSON.stringify(key)} is not [A-Za-z0-9._-]+`);
+    pairs.push(`${key}=${encode(value)}`);
+  }
+  return `[chant ${pairs.join(" ")}]`;
+}
+
+/** A comment's text with the trailer taken off. */
+export function stripMarker(comment: string): string {
+  return comment.replace(TRAILER, "");
+}
+
+/** The comment to set: the declared comment, then the trailer (with `extra` pairs, see {@link markerTrailer}). */
+export function stampedComment(declared: string | undefined, marker: OwnershipMarker | undefined, extra?: Readonly<Record<string, string>>): string {
+  const own = stripMarker(declared ?? "");
+  return own ? `${own} ${markerTrailer(marker, extra)}` : markerTrailer(marker, extra);
+}
+
+/** Every pair in a comment's trailer, decoded, or undefined when it has no trailer. */
+export function readTrailerPairs(comment: string | undefined): Map<string, string> | undefined {
+  const m = TRAILER.exec(comment ?? "");
+  if (!m) return undefined;
+  return new Map(
+    m[1]!
+      .trim()
+      .split(" ")
+      .map((p) => {
+        const at = p.indexOf("=");
+        return [p.slice(0, at), decode(p.slice(at + 1))] as const;
+      }),
+  );
+}
+
+/** Whether a comment carries chant's managed-by marker and any of `keys` in its trailer. */
+export function hasChantTrailerKey(comment: string | undefined, keys: readonly string[]): boolean {
+  const pairs = readTrailerPairs(comment);
+  return pairs?.get(COMMENT_OWNERSHIP_KEYS.managedBy) === OWNERSHIP_MANAGED_BY_VALUE && keys.some((k) => pairs!.has(k));
+}
+
+/** The marker a comment carries, or undefined when it carries none. */
+export function readMarker(comment: string | undefined): CommentMarker | undefined {
+  const pairs = readTrailerPairs(comment);
+  if (!pairs) return undefined;
+  const k = COMMENT_OWNERSHIP_KEYS;
+  const managedBy = pairs.get(k.managedBy);
+  if (managedBy === undefined) return undefined;
+  const stack = pairs.get(k.stack);
+  const env = pairs.get(k.env);
+  return { managedBy, ...(stack ? { stack } : {}), ...(env ? { env } : {}) };
+}
+
+/** Whether a comment carries chant's managed-by marker. */
+export function isChantManaged(comment: string | undefined): boolean {
+  return readMarker(comment)?.managedBy === OWNERSHIP_MANAGED_BY_VALUE;
+}
+
+/** Whether the comment's marker is exactly this project's: managed by chant, same stack, same env. */
+export function carriesMarker(comment: string | undefined, marker: OwnershipMarker | undefined): boolean {
+  const m = readMarker(comment);
+  if (m?.managedBy !== OWNERSHIP_MANAGED_BY_VALUE) return false;
+  return (m.stack ?? undefined) === (marker?.stack || undefined) && (m.env ?? undefined) === (marker?.env || undefined);
+}

package/src/core/receipts.ts

@@ -0,0 +1,29 @@
+/**
+ * Effect receipts kept on the database server itself, in every dialect
+ * (concepts/effect-receipts.mdx).
+ *
+ * A receipt is the proof that one effect ran, kept where the effect happened.
+ * A migration Op's backfill copies data on the server, so its receipts are
+ * rows on the same server, bound to the object they witness, and read and
+ * written over the connection the copy already uses. They die with the data
+ * when the Op's onFailure drops it, which a receipt kept in git would not.
+ *
+ * A dialect's store implements this interface over its own client and keeps
+ * the receipts in a table whose comment carries the ownership trailer with
+ * the `receipts` key (`./ownership.ts`), so schema reads leave it out. It is
+ * not registered as the global `receiptRead` / `receiptWrite` activities,
+ * which would take over every other lexicon's `effect()` steps in the run.
+ */
+
+import type { ReceiptStore } from "@intentius/chant/op/receipt-store";
+
+/** The receipt's address: `<stack>/<env>/<effect>`, the same fields an ownership marker carries. */
+export function receiptAddress(identity: { stack?: string; env?: string }, effect: string): string {
+  return `${identity.stack || "-"}/${identity.env || "-"}/${effect}`;
+}
+
+/** A receipt store on a database server. */
+export interface SqlReceiptStore extends ReceiptStore {
+  /** Every receipt whose address starts with `prefix`, latest value each: one query for a whole backfill. */
+  readAll(prefix: string): Promise<Map<string, string>>;
+}

package/src/core/references.ts

@@ -0,0 +1,74 @@
+/**
+ * References between schema objects, in every dialect: column lineage, the
+ * name a reference is written under in a build's output, and the order
+ * objects are created in.
+ */
+
+import type { Declarable } from "@intentius/chant";
+import type { AttrRef } from "@intentius/chant/attrref";
+import { isAttrRefLike } from "@intentius/chant/utils";
+
+/** One output column of a view and the columns it is computed from. */
+export interface LineageEdge {
+  output: string;
+  /** The select-list expression, rendered. */
+  expr: string;
+  /** The column references inside it. */
+  from: AttrRef[];
+}
+
+/** What the dependency order needs of an object: the references its DDL makes. */
+export interface Referencing {
+  readonly dependsOn: readonly unknown[];
+}
+
+/**
+ * A reference as a build's output writes it: an entity's export name, or
+ * `export.column` for a column. Undefined when the target is not exported.
+ */
+export function referenceName(value: unknown, names: Map<Declarable, string>): string | undefined {
+  if (isAttrRefLike(value)) {
+    const parent = value.parent.deref() as Declarable | undefined;
+    const owner = parent ? names.get(parent) : undefined;
+    return owner ? `${owner}.${value.attribute}` : undefined;
+  }
+  return names.get(value as Declarable);
+}
+
+/** Lineage edges with their column references written as `export.column`. */
+export function lineageJson(edges: readonly LineageEdge[], names: Map<Declarable, string>): Array<{ output: string; expr: string; from: Array<string | null> }> {
+  return edges.map((e) => ({
+    output: e.output,
+    expr: e.expr,
+    from: e.from.map((r) => referenceName(r, names) ?? null),
+  }));
+}
+
+/**
+ * Export names in creation order: an object after everything it references.
+ * Ties break by export name, so the order is the same however the files were
+ * discovered. A reference cycle is an error naming the cycle.
+ */
+export function applyOrder(objects: Map<string, Referencing>, names: Map<Declarable, string>): string[] {
+  const deps = new Map<string, string[]>();
+  for (const [name, obj] of objects) {
+    const on = new Set<string>();
+    for (const ref of obj.dependsOn) {
+      const parent = isAttrRefLike(ref) ? (ref.parent.deref() as Declarable | undefined) : (ref as Declarable);
+      const dep = parent ? names.get(parent) : undefined;
+      if (dep !== undefined && dep !== name && objects.has(dep)) on.add(dep);
+    }
+    deps.set(name, [...on].sort());
+  }
+  const order: string[] = [];
+  const done = new Set<string>();
+  const visit = (n: string, stack: string[]) => {
+    if (done.has(n)) return;
+    if (stack.includes(n)) throw new Error(`sql: a reference cycle between schema objects: ${[...stack, n].join(" -> ")}`);
+    for (const d of deps.get(n) ?? []) visit(d, [...stack, n]);
+    done.add(n);
+    order.push(n);
+  };
+  for (const n of [...objects.keys()].sort()) visit(n, []);
+  return order;
+}

package/src/core/template.ts

@@ -0,0 +1,83 @@
+/**
+ * The tagged-template plumbing every dialect's tags share: reading a
+ * template's raw parts, what a non-reference interpolation splices in, and
+ * the error a tag throws with the template line it happened on.
+ *
+ * Tokenizing and parsing the spliced statement is the dialect's own.
+ */
+
+/** A spliced value meant as a string literal. A dialect's `literal()` makes one with its own quoting. */
+export class SqlLiteral {
+  constructor(readonly sql: string) {
+    Object.freeze(this);
+  }
+}
+
+/** An error from a tag: the statement kind, the template line, and what went wrong. */
+export class SqlTemplateError extends Error {
+  constructor(
+    tag: string,
+    message: string,
+    readonly part: number,
+    readonly offset: number,
+  ) {
+    super(`${tag}\`...\`: ${message}`);
+    this.name = "SqlTemplateError";
+  }
+}
+
+/**
+ * A template's parts as the SQL reads them: the raw text, so a backslash in a
+ * regex or an escape stays as written, with the two escapes a template
+ * literal forces undone. `` \` `` is a backquote and `\${` is `${`; neither
+ * can be written in a template any other way.
+ */
+export function templateParts(strings: TemplateStringsArray | readonly string[]): string[] {
+  const raw = (strings as TemplateStringsArray).raw ?? strings;
+  return raw.map(unescapeTemplateDelimiters);
+}
+
+export function unescapeTemplateDelimiters(part: string): string {
+  return part.replace(/\\(`|\$\{)/g, "$1");
+}
+
+/** Where interpolation `index` sits in the template, for a message: the line of the template it is on. */
+export function interpolationLine(parts: readonly string[], index: number): number {
+  return parts.slice(0, index + 1).join("${}").split("\n").length;
+}
+
+/** A value's kind, for a message. */
+export function describeValue(value: unknown): string {
+  if (value === undefined) return "undefined";
+  if (value === null) return "null";
+  if (Array.isArray(value)) return "an array";
+  return typeof value === "object" ? `an object (${Object.prototype.toString.call(value)})` : `a ${typeof value}`;
+}
+
+/**
+ * The SQL text a non-reference value splices in, or an error message: a
+ * string is SQL text, a number, bigint or boolean its literal, `null` is
+ * `NULL`, a {@link SqlLiteral} its quoted form. Anything else has no SQL form.
+ */
+export function spliceText(value: unknown): string | { error: string } {
+  if (typeof value === "string") return value;
+  if (typeof value === "number") {
+    return Number.isFinite(value) ? String(value) : { error: `the number ${value} has no SQL form` };
+  }
+  if (typeof value === "bigint") return value.toString();
+  if (typeof value === "boolean") return value ? "true" : "false";
+  if (value === null) return "NULL";
+  if (value instanceof SqlLiteral) return value.sql;
+  if (value === undefined) {
+    return {
+      error:
+        "undefined. A column is referenced through `.columns` (`${events.columns.user_id}`, not " +
+        "`${events.user_id}`), and a column name that does not exist is undefined too",
+    };
+  }
+  return { error: `${describeValue(value)}, which has no SQL form` };
+}
+
+/** `o` without its `undefined` and `false` fields, so props hold only what the DDL says. */
+export const stripUnset = <T extends object>(o: T): T =>
+  Object.fromEntries(Object.entries(o).filter(([, v]) => v !== undefined && v !== false)) as T;

package/src/core/testing/http-stub.ts

@@ -0,0 +1,33 @@
+/**
+ * A stand-in HTTP server for tests, in every dialect: listens on a random
+ * loopback port and answers each request's body with what `answer` returns.
+ * A dialect's fake server (the ClickHouse HTTP interface answering its
+ * catalog reads) is built on it. Not a SQL engine.
+ */
+
+import { createServer, type IncomingMessage, type Server } from "node:http";
+
+export interface StubAnswer {
+  status: number;
+  text: string;
+}
+
+export interface HttpStub {
+  url: string;
+  close(): Promise<void>;
+}
+
+export async function startHttpStub(answer: (body: string, req: IncomingMessage) => StubAnswer): Promise<HttpStub> {
+  const server: Server = createServer((req, res) => {
+    let body = "";
+    req.on("data", (c) => (body += c));
+    req.on("end", () => {
+      const { status, text } = answer(body, req);
+      res.statusCode = status;
+      res.end(text);
+    });
+  });
+  await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r()));
+  const { port } = server.address() as { port: number };
+  return { url: `http://127.0.0.1:${port}`, close: () => new Promise((r) => server.close(() => r())) };
+}

package/src/core/tokens.ts

@@ -0,0 +1,252 @@
+/**
+ * A lossless tokenizer over a tagged template's parts (chant #3196).
+ *
+ * The template is never joined into one string with placeholders. Each literal
+ * part is tokenized on its own and an interpolation is a `ref` token between
+ * two parts, so a token's offset is an offset into the part the author wrote:
+ * what a lint rule and the editor point at.
+ *
+ * Lossless: concatenating every token's `text`, with each `ref` written back as
+ * `${...}`, gives the raw template text back byte for byte. Whitespace and
+ * comments are tokens (`ws`, `comment`), not discarded.
+ *
+ * Shared by every dialect. What differs at the character level (which quotes
+ * make an identifier, whether a backslash escapes, the punctuation and
+ * operator characters) is a dialect's {@link LexicalRules}; ClickHouse's are
+ * `../clickhouse/tokens.ts`, Postgres's `../postgres/tokens.ts`. The rules
+ * Postgres adds (#3278: dollar quotes, `E'...'` strings, nested comments,
+ * `$1` parameters, its number syntax) are optional fields, off unless a
+ * dialect sets them, so ClickHouse tokenizes exactly as before.
+ */
+
+export type TokenKind =
+  | "ws"
+  | "comment"
+  /** A bare identifier or keyword. */
+  | "ident"
+  /** A `backquoted` or "double-quoted" identifier. */
+  | "qident"
+  | "string"
+  | "number"
+  /** `( ) , ; . =` */
+  | "punct"
+  /** Any other run of operator characters. */
+  | "op"
+  /** An interpolation. */
+  | "ref"
+  /** A positional parameter, `$1` (Postgres). */
+  | "param";
+
+export interface Token {
+  kind: TokenKind;
+  text: string;
+  /** The template part the token sits in. For a `ref`, the index of the interpolated value. */
+  part: number;
+  /** Offsets into that part. A `ref` has start = end = 0. */
+  start: number;
+  end: number;
+  /**
+   * Set on a token that came from a string the author interpolated, which is
+   * spliced into the statement as SQL text: the index of that interpolation.
+   * Its offsets are into the spliced string.
+   */
+  splice?: number;
+}
+
+/** A syntax error, located in the template: the part and the offset in it. */
+export class SqlSyntaxError extends Error {
+  constructor(
+    message: string,
+    readonly part: number,
+    readonly offset: number,
+    /** The token the error is at, when there is one. A spliced token's position is the interpolation's. */
+    readonly token?: Token,
+  ) {
+    super(message);
+    this.name = "SqlSyntaxError";
+  }
+}
+
+/** What a dialect's tokenizer differs in, at the character level. */
+export interface LexicalRules {
+  /** The characters that open a quoted identifier, each closed by itself. */
+  identQuotes: string;
+  /** Whether a backslash escapes the next character inside any quote. */
+  backslashEscapes: boolean;
+  /** The single-character punctuation tokens. */
+  punct: string;
+  /** The characters an operator run is made of. */
+  opChars: RegExp;
+  /**
+   * Prefixed string constants (Postgres): `E'...'`, where a backslash escapes,
+   * `B'...'` and `X'...'` bit strings, `N'...'`, and `U&'...'` / `U&"..."`
+   * with Unicode escapes. Each is one token, its prefix included.
+   */
+  prefixedStrings?: boolean;
+  /** `$$...$$` and `$tag$...$tag$` strings, and `$1` parameters (Postgres). */
+  dollarQuotes?: boolean;
+  /** Block comments nest: a comment opened inside a block comment needs its own close (Postgres). */
+  nestedComments?: boolean;
+  /**
+   * How a number reads. `digit-names` (the default): a number may run into
+   * letters, so `00662_events` is one token, which the ClickHouse parser
+   * accepts as a name. `sql`: `42`, `3.5`, `.5`, `1e-3`, `0x1F`, `0o17`,
+   * `0b101` and `1_000_000`, and a number never runs into a name (Postgres).
+   */
+  numbers?: "digit-names" | "sql";
+  /** Identifiers may hold letters outside ASCII, as Postgres's do. */
+  unicodeIdentifiers?: boolean;
+  /**
+   * A multi-character operator cannot end in `+` or `-` unless it also holds
+   * one of `~ ! @ # % ^ & | \` ?` (Postgres, "Lexical Structure"), so
+   * `=-30` is `=` then `-30`.
+   */
+  operatorsEndWithoutSign?: boolean;
+}
+
+const IDENT_START_ASCII = /[A-Za-z_$]/;
+const IDENT_PART_ASCII = /[A-Za-z0-9_$]/;
+const IDENT_START_UNICODE = /[A-Za-z_\u0080-\uffff]/;
+const IDENT_PART_UNICODE = /[A-Za-z0-9_$\u0080-\uffff]/;
+
+/** Tokenize one string. `part` is the part index recorded on each token and on an error. */
+export function tokenizeText(src: string, part: number, rules: LexicalRules): Token[] {
+  const tokens: Token[] = [];
+  let i = 0;
+  const push = (kind: TokenKind, start: number, end: number) =>
+    tokens.push({ kind, text: src.slice(start, end), part, start, end });
+  const identStart = rules.unicodeIdentifiers ? IDENT_START_UNICODE : IDENT_START_ASCII;
+  const identPart = rules.unicodeIdentifiers ? IDENT_PART_UNICODE : IDENT_PART_ASCII;
+
+  /** The index after the quote that closes the one at `at`. A doubled quote is an escaped quote. */
+  const quoted = (at: number, q: string, backslash: boolean): number => {
+    let j = at + 1;
+    for (;;) {
+      if (j >= src.length) throw new SqlSyntaxError(`unterminated ${q} quote`, part, at);
+      if (backslash && src[j] === "\\") {
+        j += 2;
+        continue;
+      }
+      if (src[j] === q) {
+        if (src[j + 1] === q) {
+          j += 2;
+          continue;
+        }
+        return j + 1;
+      }
+      j++;
+    }
+  };
+
+  while (i < src.length) {
+    const c = src[i]!;
+    const start = i;
+    if (/\s/.test(c)) {
+      while (i < src.length && /\s/.test(src[i]!)) i++;
+      push("ws", start, i);
+    } else if (c === "-" && src[i + 1] === "-") {
+      while (i < src.length && src[i] !== "\n") i++;
+      push("comment", start, i);
+    } else if (c === "/" && src[i + 1] === "*") {
+      if (rules.nestedComments) {
+        let depth = 1;
+        i += 2;
+        while (depth > 0) {
+          if (i >= src.length) throw new SqlSyntaxError("unterminated /* comment", part, start);
+          if (src[i] === "*" && src[i + 1] === "/") {
+            depth--;
+            i += 2;
+          } else if (src[i] === "/" && src[i + 1] === "*") {
+            depth++;
+            i += 2;
+          } else i++;
+        }
+      } else {
+        const close = src.indexOf("*/", i + 2);
+        if (close < 0) throw new SqlSyntaxError("unterminated /* comment", part, start);
+        i = close + 2;
+      }
+      push("comment", start, i);
+    } else if (rules.prefixedStrings && /[EeBbXxNn]/.test(c) && src[i + 1] === "'" && !(start > 0 && identPart.test(src[start - 1]!))) {
+      i = quoted(i + 1, "'", c === "E" || c === "e");
+      push("string", start, i);
+    } else if (rules.prefixedStrings && /[Uu]/.test(c) && src[i + 1] === "&" && (src[i + 2] === "'" || src[i + 2] === '"') && !(start > 0 && identPart.test(src[start - 1]!))) {
+      const q = src[i + 2]!;
+      i = quoted(i + 2, q, false);
+      push(q === "'" ? "string" : "qident", start, i);
+    } else if (c === "'" || rules.identQuotes.includes(c)) {
+      i = quoted(i, c, rules.backslashEscapes);
+      push(c === "'" ? "string" : "qident", start, i);
+    } else if (rules.dollarQuotes && c === "$" && /[0-9]/.test(src[i + 1] ?? "")) {
+      i++;
+      while (i < src.length && /[0-9]/.test(src[i]!)) i++;
+      push("param", start, i);
+    } else if (rules.dollarQuotes && c === "$" && /^\$([A-Za-z_\u0080-\uffff][A-Za-z0-9_\u0080-\uffff]*)?\$/.test(src.slice(i))) {
+      const delimiter = /^\$([A-Za-z_\u0080-\uffff][A-Za-z0-9_\u0080-\uffff]*)?\$/.exec(src.slice(i))![0];
+      const close = src.indexOf(delimiter, i + delimiter.length);
+      if (close < 0) throw new SqlSyntaxError(`unterminated ${delimiter} string`, part, start);
+      i = close + delimiter.length;
+      push("string", start, i);
+    } else if (rules.numbers === "sql" && (/[0-9]/.test(c) || (c === "." && /[0-9]/.test(src[i + 1] ?? "")))) {
+      if (/^0[xXoObB]/.test(src.slice(i, i + 2))) {
+        i += 2;
+        while (i < src.length && /[0-9A-Fa-f_]/.test(src[i]!)) i++;
+      } else {
+        while (i < src.length && /[0-9_]/.test(src[i]!)) i++;
+        if (src[i] === "." && src[i + 1] !== ".") {
+          i++;
+          while (i < src.length && /[0-9_]/.test(src[i]!)) i++;
+        }
+        if (/[eE]/.test(src[i] ?? "") && /^([0-9]|[+-][0-9])/.test(src.slice(i + 1, i + 3))) {
+          i += 2;
+          while (i < src.length && /[0-9]/.test(src[i]!)) i++;
+        }
+      }
+      push("number", start, i);
+    } else if (rules.numbers !== "sql" && /[0-9]/.test(c)) {
+      // `1.5`, `1e3`, `0x1F`; and a name that starts with a digit (`00662_events`), which the parser accepts as a name.
+      while (i < src.length && /[0-9a-zA-Z_.]/.test(src[i]!)) {
+        if (src[i] === "." && !/[0-9]/.test(src[i + 1] ?? "")) break;
+        i++;
+      }
+      push("number", start, i);
+    } else if (identStart.test(c)) {
+      while (i < src.length && identPart.test(src[i]!)) i++;
+      push("ident", start, i);
+    } else if (rules.punct.includes(c)) {
+      i++;
+      push("punct", start, i);
+    } else {
+      while (
+        i < src.length &&
+        rules.opChars.test(src[i]!) &&
+        !(src[i] === "-" && src[i + 1] === "-") &&
+        !(rules.nestedComments && src[i] === "/" && src[i + 1] === "*")
+      )
+        i++;
+      if (i === start) i++;
+      if (rules.operatorsEndWithoutSign && i - start > 1 && !/[~!@#%^&|`?]/.test(src.slice(start, i))) {
+        while (i - start > 1 && /[+-]/.test(src[i - 1]!)) i--;
+      }
+      push("op", start, i);
+    }
+  }
+  return tokens;
+}
+
+/** Tokenize a template's parts, with a `ref` token for each interpolation between them. */
+export function tokenize(parts: readonly string[], rules: LexicalRules): Token[] {
+  const tokens: Token[] = [];
+  parts.forEach((src, part) => {
+    tokens.push(...tokenizeText(src, part, rules));
+    if (part < parts.length - 1) tokens.push({ kind: "ref", text: "", part, start: 0, end: 0 });
+  });
+  return tokens;
+}
+
+export const isTrivia = (t: Token): boolean => t.kind === "ws" || t.kind === "comment";
+
+/** Every token's text back in order, each `ref` written by `ref`. */
+export function untokenize(tokens: readonly Token[], ref: (index: number) => string): string {
+  return tokens.map((t) => (t.kind === "ref" ? ref(t.part) : t.text)).join("");
+}

package/src/coverage.ts

@@ -0,0 +1,64 @@
+/**
+ * How much of the pinned ClickHouse catalog the dialect types beyond its name.
+ *
+ * Every engine, type family, codec, skip index type and setting at the pin is
+ * generated as a name, so name coverage is total by construction. What varies
+ * is how far the overlays go: an engine whose arguments have kinds can be
+ * checked, one whose arguments are `unknown` can only be named. This reports
+ * that, per section, from the generated tables.
+ */
+
+import { CODECS, SKIP_INDEX_TYPES, TABLE_ENGINES, CLICKHOUSE_VERSION } from "./generated/clickhouse";
+
+export interface CoverageSection {
+  section: string;
+  total: number;
+  typed: number;
+  untyped: string[];
+}
+
+export interface CoverageReport {
+  version: string;
+  sections: CoverageSection[];
+}
+
+export function analyze(): CoverageReport {
+  const engines = Object.entries(TABLE_ENGINES).filter(([, e]) => e.args !== undefined);
+  const mergeTree = engines.filter(([, e]) => e.mergeTree);
+  const codecs = Object.entries(CODECS).filter(([, c]) => c.role !== "internal");
+  const indexes = Object.entries(SKIP_INDEX_TYPES);
+
+  const section = (name: string, rows: Array<[string, boolean]>): CoverageSection => ({
+    section: name,
+    total: rows.length,
+    typed: rows.filter(([, ok]) => ok).length,
+    untyped: rows.filter(([, ok]) => !ok).map(([n]) => n),
+  });
+
+  return {
+    version: CLICKHOUSE_VERSION,
+    sections: [
+      section("MergeTree engine arguments", mergeTree.map(([n, e]) => [n, e.typed])),
+      section("all engine arguments", engines.map(([n, e]) => [n, e.typed])),
+      section("codec parameters", codecs.map(([n, c]) => [n, c.parameters !== "unknown"])),
+      section(
+        "skip index parameters",
+        indexes.map(([n, i]) => [n, (i.args ?? []).every((a) => a.kind !== "unknown")]),
+      ),
+    ],
+  };
+}
+
+export function printCoverage(report: CoverageReport, opts: { verbose?: boolean; minOverall?: number } = {}): void {
+  console.error(`ClickHouse ${report.version}`);
+  for (const s of report.sections) {
+    const pct = s.total === 0 ? 100 : Math.round((s.typed / s.total) * 100);
+    console.error(`  ${s.section}: ${s.typed}/${s.total} typed (${pct}%)`);
+    if (opts.verbose && s.untyped.length > 0) console.error(`    untyped: ${s.untyped.join(", ")}`);
+  }
+  if (opts.minOverall !== undefined) {
+    const mergeTree = report.sections[0]!;
+    const pct = mergeTree.total === 0 ? 100 : (mergeTree.typed / mergeTree.total) * 100;
+    if (pct < opts.minOverall) throw new Error(`MergeTree argument coverage ${pct.toFixed(0)}% is below ${opts.minOverall}%`);
+  }
+}