@intentius/chant-lexicon-sql 0.124.31 → 0.124.32

package/dist/postgres/testing/writable-server.js

@@ -0,0 +1,160 @@
+/**
+ * A fake Postgres server for the applier's unit tests (#3280): a client that
+ * keeps the objects it was sent `CREATE` for, their comments set by
+ * `COMMENT ON`, and drops them on `DROP`, with transactions that roll back;
+ * and a catalog reader over what it keeps, standing in for
+ * `../live/catalog.ts`. What it was sent is recorded, so a test asserts on
+ * the transport.
+ *
+ * It parses statements with the dialect's own parser, so what it stores reads
+ * back as the declaration it came from. It does not model locks, column
+ * changes or dependencies: a test that needs those runs against the pinned
+ * server (`../../op/activities/postgres-apply.e2e.test.ts`).
+ */
+import { PostgresQueryError } from "../live/client.js";
+import { stripMarker } from "../../core/ownership.js";
+import { tokenizeText } from "../tokens.js";
+import { parseStatements } from "../parser.js";
+import { POSTGRES_ENTITY_TYPES } from "../entity-types.js";
+import { quoteIdent } from "../keywords.js";
+const KIND_WORD = {
+    [POSTGRES_ENTITY_TYPES.schema]: "SCHEMA",
+    [POSTGRES_ENTITY_TYPES.table]: "TABLE",
+    [POSTGRES_ENTITY_TYPES.index]: "INDEX",
+    [POSTGRES_ENTITY_TYPES.view]: "VIEW",
+    [POSTGRES_ENTITY_TYPES.materializedView]: "MATERIALIZED VIEW",
+    [POSTGRES_ENTITY_TYPES.sequence]: "SEQUENCE",
+    [POSTGRES_ENTITY_TYPES.enum]: "TYPE",
+    [POSTGRES_ENTITY_TYPES.domain]: "DOMAIN",
+    [POSTGRES_ENTITY_TYPES.extension]: "EXTENSION",
+};
+const flat = (type) => type === POSTGRES_ENTITY_TYPES.schema || type === POSTGRES_ENTITY_TYPES.extension;
+const qname = (o) => (flat(o.type) || !o.schema ? quoteIdent(o.name) : `${quoteIdent(o.schema)}.${quoteIdent(o.name)}`);
+const split = (n, defaultSchema = "public") => n.pieces.length > 1 ? { schema: n.pieces[0], name: n.pieces[1] } : { schema: defaultSchema, name: n.pieces[0] };
+const unquote = (s) => s.replace(/^'/, "").replace(/'$/, "").replace(/''/g, "'");
+export function writablePostgres(seed = []) {
+    let objects = seed.map((o) => ({ ...o }));
+    let snapshot;
+    const server = {
+        objects,
+        writes: [],
+        deletes: [],
+        control: [],
+        readLive: async () => server.objects.map((o, i) => {
+            const own = o.comment !== undefined ? stripMarker(o.comment) : "";
+            return {
+                type: o.type,
+                ...(flat(o.type) ? {} : { schema: o.schema }),
+                name: o.name,
+                oid: String(16384 + i),
+                ...(o.comment !== undefined ? { comment: o.comment } : {}),
+                ...(o.foreign ? { foreign: o.foreign } : {}),
+                statement: [o.create, ...(own ? [`COMMENT ON ${KIND_WORD[o.type]} ${qname(o)} IS '${own.replace(/'/g, "''")}'`] : [])].join(";\n"),
+            };
+        }),
+        client: {
+            async query(sql) {
+                const s = sql.trim();
+                if (/^BEGIN$/i.test(s)) {
+                    server.control.push("BEGIN");
+                    snapshot = server.objects.map((o) => ({ ...o }));
+                    return [];
+                }
+                if (/^COMMIT$/i.test(s)) {
+                    server.control.push("COMMIT");
+                    snapshot = undefined;
+                    return [];
+                }
+                if (/^ROLLBACK$/i.test(s)) {
+                    server.control.push("ROLLBACK");
+                    if (snapshot)
+                        server.objects = objects = snapshot;
+                    snapshot = undefined;
+                    return [];
+                }
+                if (/^(SET|SELECT|RESET)\b/i.test(s))
+                    return [];
+                if (server.refuse?.test(s))
+                    throw new PostgresQueryError(`refused: ${s.split("\n")[0]}`, server.refuseCode ?? "XX000");
+                server.writes.push(s);
+                const drop = /^DROP\s+(MATERIALIZED VIEW|INDEX(?:\s+CONCURRENTLY)?|TABLE|VIEW|SEQUENCE|TYPE|DOMAIN|EXTENSION|SCHEMA)\s+(.+)$/i.exec(s);
+                if (drop) {
+                    server.deletes.push(s);
+                    const word = drop[1].toUpperCase().replace(/\s+CONCURRENTLY$/, "");
+                    const at = server.objects.findIndex((o) => KIND_WORD[o.type] === word && qname(o) === drop[2].trim());
+                    if (at >= 0)
+                        server.objects.splice(at, 1);
+                    return [];
+                }
+                let node;
+                try {
+                    node = parseStatements(tokenizeText(s, 0))[0];
+                }
+                catch {
+                    return []; // ALTER and the rest: recorded, not modelled
+                }
+                if (!node)
+                    return [];
+                if (node.statement === "comment") {
+                    const c = node;
+                    const entry = Object.entries(KIND_WORD).find(([, w]) => w === c.objectType.toUpperCase());
+                    if (!entry)
+                        return [];
+                    const type = entry[0];
+                    const n = flat(type) ? { name: c.target.pieces[0] } : split(c.target);
+                    const o = server.objects.find((x) => x.type === type && x.name === n.name && (flat(type) || x.schema === n.schema));
+                    const text = c.text ? unquote(/IS\s+('(?:[^']|'')*')\s*$/i.exec(s)?.[1] ?? "") : undefined;
+                    if (o) {
+                        if (text === undefined)
+                            delete o.comment;
+                        else
+                            o.comment = text;
+                    }
+                    return [];
+                }
+                const add = (type, schema, name) => {
+                    if (server.objects.some((x) => x.type === type && x.name === name && x.schema === schema)) {
+                        throw new PostgresQueryError(`relation "${name}" already exists`, "42P07");
+                    }
+                    server.objects.push({ type, ...(schema !== undefined ? { schema } : {}), name, create: s.replace(/\bCREATE\s+OR\s+REPLACE\s+/i, "CREATE ") });
+                };
+                switch (node.statement) {
+                    case "schema":
+                        add(POSTGRES_ENTITY_TYPES.schema, undefined, node.name.pieces[0]);
+                        break;
+                    case "extension":
+                        add(POSTGRES_ENTITY_TYPES.extension, undefined, node.name.pieces[0]);
+                        break;
+                    case "table":
+                    case "sequence":
+                    case "enum":
+                    case "domain": {
+                        const n = split(node.name);
+                        const type = { table: POSTGRES_ENTITY_TYPES.table, sequence: POSTGRES_ENTITY_TYPES.sequence, enum: POSTGRES_ENTITY_TYPES.enum, domain: POSTGRES_ENTITY_TYPES.domain }[node.statement];
+                        add(type, n.schema, n.name);
+                        break;
+                    }
+                    case "view": {
+                        const n = split(node.name);
+                        const type = node.materialized ? POSTGRES_ENTITY_TYPES.materializedView : POSTGRES_ENTITY_TYPES.view;
+                        if (node.orReplace) {
+                            const o = server.objects.find((x) => x.type === type && x.name === n.name && x.schema === n.schema);
+                            if (o) {
+                                o.create = s.replace(/\bCREATE\s+OR\s+REPLACE\s+/i, "CREATE ");
+                                break;
+                            }
+                        }
+                        add(type, n.schema, n.name);
+                        break;
+                    }
+                    case "index":
+                        add(POSTGRES_ENTITY_TYPES.index, split(node.table).schema, node.name.pieces[0]);
+                        break;
+                }
+                return [];
+            },
+            async end() { },
+        },
+    };
+    return server;
+}

package/dist/postgres/tokens.js

@@ -0,0 +1,41 @@
+/**
+ * Postgres's lexical rules over the shared lossless tokenizer
+ * (`../core/tokens.ts`, chant #3278), from the Postgres 18 reference,
+ * "Lexical Structure":
+ *
+ * - `"` quotes an identifier; a backtick is an operator character.
+ * - A backslash is an ordinary character in `'...'` (standard_conforming_strings
+ *   is on by default since 9.1) and escapes only in an `E'...'` string. `B'...'`,
+ *   `X'...'`, `N'...'` and `U&'...'` are strings too, each one token.
+ * - `$$...$$` and `$tag$...$tag$` are strings; `$1` is a parameter.
+ * - Block comments nest.
+ * - `( ) , ; . [ ]` are punctuation; `=` is an operator, so `WITH (a = 1)`
+ *   reads `=` as an `op` token.
+ * - A number never runs into a name: `42`, `3.5`, `.5`, `1e-3`, `0x1F`,
+ *   `0o17`, `0b101`, `1_000`.
+ * - A multi-character operator does not end in `+` or `-` unless it holds
+ *   one of `~ ! @ # % ^ & | \` ?`, so `fillfactor=-30` reads `=` then `-`.
+ */
+import { tokenize as tokenizeWith, tokenizeText as tokenizeTextWith } from "../core/tokens.js";
+export { isTrivia, SqlSyntaxError, untokenize } from "../core/tokens.js";
+export const POSTGRES_LEXICAL = {
+    identQuotes: '"',
+    backslashEscapes: false,
+    punct: "(),;.[]",
+    // The operator characters (sql-syntax-lexical, "Operators"), and `:` for the `::` cast.
+    opChars: /[+\-*/<>=~!@#%^&|`?:]/,
+    prefixedStrings: true,
+    dollarQuotes: true,
+    nestedComments: true,
+    numbers: "sql",
+    unicodeIdentifiers: true,
+    operatorsEndWithoutSign: true,
+};
+/** Tokenize one string with Postgres's rules. `part` is the part index recorded on each token and on an error. */
+export function tokenizeText(src, part) {
+    return tokenizeTextWith(src, part, POSTGRES_LEXICAL);
+}
+/** Tokenize a template's parts with Postgres's rules, with a `ref` token for each interpolation between them. */
+export function tokenize(parts) {
+    return tokenizeWith(parts, POSTGRES_LEXICAL);
+}

package/dist/postgres.js

@@ -0,0 +1,15 @@
+/**
+ * `@intentius/chant-lexicon-sql/postgres`: the Postgres dialect (#3289).
+ *
+ * Everything here is spec-true to Postgres, not a database-agnostic model:
+ * the statements are Postgres's own DDL, parsed at build time into entities,
+ * and the names fold and quote as Postgres folds and quotes them.
+ */
+export { schema, table, index, view, sequence, type, domain, extension, func, procedure, aggregate, trigger, eventTrigger, collation, cast, policy, role, grant, literal, SqlLiteral, SqlTemplateError, PostgresObject, POSTGRES_ENTITY_TYPES, isPostgresObject, isColumnRef, quoteIdent, } from "./postgres/entities.js";
+export { POSTGRES_PROVIDERS, COMMON_REFUSED, providerData, isPostgresProvider, providerAllowsExtension, isProviderOwned, refusedStatement, normalizeStatement, } from "./postgres/providers/index.js";
+export { PostgresMigrationOp, } from "./postgres/migrate/op.js";
+export { POSTGRES_RECEIPTS_TABLE } from "./postgres/migrate/receipts.js";
+export * from "./composites/postgres.js";
+// The build's Postgres objects as typed values, for a project's own
+// `lint.policies` (#3870); also at `@intentius/chant-lexicon-sql/read`.
+export { readPostgres, canonicalPostgresType, } from "./read/postgres.js";

package/dist/read/clickhouse.js

@@ -0,0 +1,82 @@
+/**
+ * The ClickHouse half of the public read API (`../read.ts`): the build's
+ * tables, columns, sort keys, skip indexes, engines and views as typed values,
+ * built from the serialized output. The lexicon's post-synth checks read the
+ * same values (`../lint/post-synth/clickhouse-helpers.ts`).
+ */
+import { relationKey } from "../core/typed.js";
+import { bareName, baseEngineName, isMergeTree, isNullableType, keyElements, typeFamily, unwrapType } from "../clickhouse/expressions.js";
+import { dialectObjects } from "./objects.js";
+function columnInfo(c, sortKey) {
+    const inner = /^LowCardinality\s*\(/i.test((c.type ?? "").trim());
+    const nullable = c.nullable === true || isNullableType(c.type) || (inner && isNullableType(unwrapLowCardinality(c.type)));
+    return {
+        ...c,
+        nullable,
+        lowCardinality: inner,
+        ...(c.type !== undefined ? { baseType: unwrapType(c.type), family: typeFamily(c.type) } : {}),
+        inSortKey: sortKey.has(c.name),
+    };
+}
+function unwrapLowCardinality(type) {
+    return type.trim().replace(/^LowCardinality\s*\(([\s\S]*)\)$/i, "$1").trim();
+}
+function build(objects) {
+    const tables = objects
+        .filter((o) => o.type === "ClickHouse::Table")
+        .map((o) => {
+        const t = o;
+        const sortKey = t.orderBy ? keyElements(t.orderBy) : [];
+        const bare = new Set(sortKey.map(bareName).filter((n) => n !== undefined));
+        const columns = (t.columns ?? []).map((c) => columnInfo(c, bare));
+        return {
+            ...t,
+            qualifiedName: relationKey(t),
+            columns,
+            ...(t.engine ? { engineFamily: baseEngineName(t.engine) } : {}),
+            mergeTree: isMergeTree(t.engine),
+            sortKey,
+            primaryKeyElements: t.primaryKey ? keyElements(t.primaryKey) : sortKey,
+            column: (name) => columns.find((c) => c.name === name),
+        };
+    });
+    const views = objects
+        .filter((o) => o.type === "ClickHouse::View" || o.type === "ClickHouse::MaterializedView")
+        .map((o) => {
+        const v = o;
+        const columns = (v.columns ?? []).map((c) => columnInfo(c, new Set()));
+        return {
+            ...v,
+            qualifiedName: relationKey(v),
+            materialized: v.type === "ClickHouse::MaterializedView",
+            columns,
+            outputs: columns.length > 0 ? columns.map((c) => c.name) : (v.lineage ?? []).map((e) => e.output),
+        };
+    });
+    return {
+        dialect: "clickhouse",
+        tables,
+        views,
+        objects,
+        table: (ref) => tables.find((t) => t.export === ref) ?? tables.find((t) => t.qualifiedName === ref) ?? tables.find((t) => t.name === ref),
+    };
+}
+const cache = new WeakMap();
+/**
+ * What a ClickHouse build declares, read from a post-synth context's `docs`
+ * (a `lint.policies` check's `ctx`), or from the serialized output document
+ * itself. Empty when the build has no ClickHouse output. The result is cached
+ * per context; treat it as read-only.
+ */
+export function readClickHouse(source) {
+    if ("objects" in source)
+        return build(source.dialect === "clickhouse" ? source.objects : []);
+    const key = source.docs;
+    const hit = key ? cache.get(key) : undefined;
+    if (hit)
+        return hit;
+    const read = build(dialectObjects(source, "clickhouse"));
+    if (key)
+        cache.set(key, read);
+    return read;
+}

package/dist/read/objects.js

@@ -0,0 +1,44 @@
+/**
+ * The sql build output as the serializer writes it (`../serializer.ts`): one
+ * JSON document per build with `dialect`, `applyOrder` and `objects`, each
+ * object its entity's props with the template source dropped, the DDL moved
+ * to the end, and every reference to another declared object written as that
+ * object's export name.
+ *
+ * This is the one place that finds those documents in a post-synth context.
+ * The public read API (`../read.ts`) and the lexicon's own post-synth checks
+ * (`../lint/post-synth/sql-helpers.ts`) both start here.
+ */
+// ── Finding the documents ─────────────────────────────────────────────
+/** True for a value shaped like the serializer's document. */
+export function isSqlOutputDocument(value) {
+    const v = value;
+    return v !== null && typeof v === "object" && (v.dialect === "clickhouse" || v.dialect === "postgres") && Array.isArray(v.objects);
+}
+/** Every sql output document in the context, in output order. */
+export function sqlDocuments(source) {
+    const out = [];
+    for (const doc of source.docs ?? []) {
+        if (doc.error || doc.lexicon !== "sql" || doc.format !== "json")
+            continue;
+        if (isSqlOutputDocument(doc.value))
+            out.push(doc.value);
+    }
+    return out;
+}
+/** Every serialized object of one dialect, across the context's sql documents. */
+export function dialectObjects(source, dialect) {
+    const out = [];
+    for (const doc of sqlDocuments(source))
+        if (doc.dialect === dialect)
+            out.push(...doc.objects);
+    return out;
+}
+/** The Postgres major the build output records, else `fallback`. */
+export function recordedPostgresMajor(source, fallback) {
+    for (const doc of sqlDocuments(source)) {
+        if (doc.dialect === "postgres" && typeof doc.postgresMajor === "number")
+            return doc.postgresMajor;
+    }
+    return fallback;
+}

package/dist/read/postgres.js

@@ -0,0 +1,104 @@
+/**
+ * The Postgres half of the public read API (`../read.ts`): the build's
+ * tables, columns, keys, indexes and views as typed values, built from the
+ * serialized output. The lexicon's post-synth checks read the same values
+ * (`../lint/post-synth/postgres-helpers.ts`).
+ */
+import { POSTGRES_MAJORS } from "../generated/postgres.js";
+import { relationKey } from "../core/typed.js";
+import { parseType, projectFromObjects, resolveType, typeLabel } from "../lint/post-synth/postgres-names.js";
+import { dialectObjects, recordedPostgresMajor, } from "./objects.js";
+const PINNED_MAJOR = POSTGRES_MAJORS[POSTGRES_MAJORS.length - 1];
+/** A column type's canonical name, array-ness and serial-ness against a project. */
+function typeOf(p, text) {
+    const t = parseType(text);
+    if (!t)
+        return { ...(text ? { typeName: text.trim().toLowerCase() } : {}), array: false, serial: false };
+    const r = resolveType(p, t, { serial: true });
+    const typeName = r.kind === "builtin" || r.kind === "absent" ? r.canonical : r.kind === "declared" ? String(r.object.sqlName ?? r.object.name) : typeLabel(t);
+    return { typeName, array: t.array, serial: r.kind === "builtin" && r.serial === true };
+}
+/**
+ * The canonical name of a Postgres type as written: `timestamptz` and
+ * `TIMESTAMP WITH TIME ZONE` are both `timestamp with time zone`, `int4` is
+ * `integer`. Compare a column's {@link PostgresColumnInfo.typeName} with it,
+ * so a rule does not depend on how the DDL spelled the type.
+ */
+export function canonicalPostgresType(text) {
+    return typeOf(projectFromObjects([], PINNED_MAJOR), text).typeName;
+}
+function columnInfo(p, c, pk) {
+    const { typeName, array, serial } = typeOf(p, c.type);
+    const inPrimaryKey = pk.has(c.name);
+    const nullable = !(c.notNull === true || inPrimaryKey || c.generated?.kind === "identity" || serial);
+    return { ...c, nullable, ...(typeName !== undefined ? { typeName } : {}), array, inPrimaryKey };
+}
+function indexInfo(i) {
+    const columns = [];
+    for (const e of i.elements ?? []) {
+        if (!e.column)
+            break;
+        columns.push(e.column);
+    }
+    return { ...i, columns };
+}
+function build(objects, major) {
+    const p = projectFromObjects(objects, major);
+    const indexes = objects.filter((o) => o.type === "Postgres::Index").map((o) => indexInfo(o));
+    const tables = objects
+        .filter((o) => o.type === "Postgres::Table")
+        .map((o) => {
+        const t = o;
+        const pk = new Set(t.primaryKey?.columns ?? []);
+        const columns = (t.columns ?? []).map((c) => columnInfo(p, c, pk));
+        return {
+            ...t,
+            qualifiedName: relationKey(t),
+            columns,
+            indexes: indexes.filter((i) => i.table === t.export || i.tableName === t.sqlName),
+            column: (name) => columns.find((c) => c.name === name),
+        };
+    });
+    const views = objects
+        .filter((o) => o.type === "Postgres::View" || o.type === "Postgres::MaterializedView")
+        .map((o) => {
+        const v = o;
+        const declared = v.columns ?? [];
+        return {
+            ...v,
+            qualifiedName: relationKey(v),
+            materialized: v.type === "Postgres::MaterializedView",
+            outputs: declared.length > 0 ? [...declared] : (v.lineage ?? []).map((e) => e.output),
+        };
+    });
+    return {
+        dialect: "postgres",
+        major,
+        tables,
+        views,
+        indexes,
+        objects,
+        table: (ref) => tables.find((t) => t.export === ref) ?? tables.find((t) => t.qualifiedName === ref) ?? tables.find((t) => t.name === ref),
+    };
+}
+const cache = new WeakMap();
+/**
+ * What a Postgres build declares, read from a post-synth context's `docs`
+ * (a `lint.policies` check's `ctx`), or from the serialized output document
+ * itself. Empty when the build has no Postgres output. The result is cached
+ * per context; treat it as read-only.
+ */
+export function readPostgres(source) {
+    if ("objects" in source) {
+        const objects = source.dialect === "postgres" ? source.objects : [];
+        return build(objects, typeof source.postgresMajor === "number" ? source.postgresMajor : PINNED_MAJOR);
+    }
+    const key = source.docs;
+    const hit = key ? cache.get(key) : undefined;
+    if (hit)
+        return hit;
+    const read = build(dialectObjects(source, "postgres"), recordedPostgresMajor(source, PINNED_MAJOR));
+    if (key)
+        cache.set(key, read);
+    return read;
+}

package/dist/read.js

@@ -0,0 +1,52 @@
+/**
+ * `@intentius/chant-lexicon-sql/read`: a typed, read-only view of the SQL
+ * objects a build declares (#3870), for a project's own `lint.policies`.
+ *
+ * A policy's `check(ctx)` gets the build's parsed output in `ctx.docs`.
+ * {@link readPostgres} and {@link readClickHouse} turn the sql lexicon's part
+ * of it into tables with typed columns (SQL type, nullability, default,
+ * comment), keys, indexes and engines, and views with their outputs and
+ * column lineage. The lexicon's own post-synth checks read the same values,
+ * so a project rule sees exactly what `SQLPG101` or `SQLCH113` sees.
+ *
+ * ```ts
+ * import type { PostSynthCheck } from "@intentius/chant/lint/post-synth";
+ * import { canonicalPostgresType, readPostgres } from "@intentius/chant-lexicon-sql/read";
+ *
+ * const TIMESTAMPTZ = canonicalPostgresType("timestamptz");
+ *
+ * export const createdAt: PostSynthCheck = {
+ *   id: "ORG-CREATED-AT",
+ *   description: "every table has created_at timestamptz NOT NULL",
+ *   check(ctx) {
+ *     return readPostgres(ctx).tables
+ *       .filter((t) => { const c = t.column("created_at"); return !c || c.typeName !== TIMESTAMPTZ || c.nullable; })
+ *       .map((t) => ({ checkId: "ORG-CREATED-AT", severity: "error", entity: t.export,
+ *                      message: `${t.qualifiedName} needs created_at timestamptz NOT NULL` }));
+ *   },
+ * };
+ * ```
+ *
+ * Every value keeps the fields the serializer writes (see the Serialization
+ * page) and adds the derived ones documented on each type. The reads also take
+ * the serialized document itself (`{ dialect, objects }`), for a tool that
+ * holds a recorded build rather than a post-synth context.
+ */
+export { readPostgres, canonicalPostgresType } from "./read/postgres.js";
+export { readClickHouse } from "./read/clickhouse.js";
+export { sqlDocuments, isSqlOutputDocument } from "./read/objects.js";
+import { readClickHouse } from "./read/clickhouse.js";
+import { readPostgres } from "./read/postgres.js";
+import { sqlDocuments } from "./read/objects.js";
+/**
+ * The build's SQL objects in whichever dialect it declares: one build holds
+ * one dialect. Undefined when the build has no sql output.
+ */
+export function readSql(source) {
+    const dialect = "objects" in source ? source.dialect : sqlDocuments(source)[0]?.dialect;
+    if (dialect === "postgres")
+        return readPostgres(source);
+    if (dialect === "clickhouse")
+        return readClickHouse(source);
+    return undefined;
+}