@intentius/chant-lexicon-sql 0.124.46 → 0.124.48

package/dist/clickhouse/review-catalog.js

@@ -0,0 +1,330 @@
+/**
+ * The needs-review catalog: the judgment calls that turn a table from another
+ * database into a ClickHouse table, each as a finding with the evidence it
+ * used. A source reader (`chant import` of SQL Server, #4398; `chant migrate`
+ * of Postgres, #4399) does the mechanical work, a type map and a rename,
+ * and asks this module for everything that is a decision.
+ *
+ * Nothing here knows a source dialect. A reader describes its table in the
+ * neutral facts below (a candidate sort key, a column's nullability, a
+ * constraint) and gets back the ClickHouse choice, a finding to report, and
+ * for a constraint the comment lines to keep in the DDL. The rules are the
+ * same whichever database the table came from.
+ *
+ * ## API
+ *
+ * - {@link sortKeyReview}: `ORDER BY` from the candidates the source offers,
+ *   in the priority `explicit-order`, `clustering-key`, `primary-key`,
+ *   `unique-key`, then `tuple()`. Returns `orderBy`, the `settings` the key
+ *   needs (`allow_nullable_key` when a key column is `Nullable`), and a
+ *   finding that names the chosen candidate and the others it passed over.
+ * - {@link engineReview}: `MergeTree` for an append-only table;
+ *   `ReplacingMergeTree(_version, _deleted)` for a CDC mirror, with the two
+ *   columns it adds in `extraColumns`. `cdcColumns` names them otherwise
+ *   (the Postgres pair uses the `CdcMirror` composite's `_is_deleted`).
+ * - {@link nullableReview}: a column's final type. A `NOT NULL` column keeps
+ *   its type with no finding. A nullable column becomes `Nullable(T)` with a
+ *   finding; when it has a default the finding says the default could
+ *   replace `Nullable`.
+ * - {@link lowCardinalityReview}: a finding for a `String` column whose name
+ *   and declared length suggest few distinct values. It never changes the
+ *   type; offline, the name and the type are all there is to read.
+ * - {@link constraintReview}: foreign keys, checks, unique and exclusion
+ *   constraints, which ClickHouse does not enforce. Returns `comments` (SQL `--` lines to
+ *   put before the `CREATE`) and one finding per constraint.
+ * - {@link REVIEW_RULES}: every rule id with its summary and the ClickHouse
+ *   documentation it rests on, for report metadata.
+ * - {@link findingText}: a finding as one line (object, message, evidence,
+ *   suggestion), the form a SARIF `message` takes.
+ *
+ * A finding's `severity` is `warning` for a decision the author must confirm
+ * and `info` for a note that changes nothing. The caller maps findings to its
+ * own diagnostic type; `at` carries the source line and column when the
+ * reader knows them.
+ *
+ * Every ClickHouse type and engine this module writes is checked against the
+ * pinned catalog (`src/generated/clickhouse.ts`, ClickHouse 26.8) in
+ * `review-catalog.test.ts`, so a pin move that drops one fails there.
+ *
+ * Sources: the MergeTree sorting key and `allow_nullable_key`
+ * (https://clickhouse.com/docs/engines/table-engines/mergetree-family/mergetree),
+ * `ReplacingMergeTree(ver, is_deleted)`
+ * (https://clickhouse.com/docs/engines/table-engines/mergetree-family/replacingmergetree),
+ * `Nullable` (https://clickhouse.com/docs/sql-reference/data-types/nullable),
+ * `LowCardinality`
+ * (https://clickhouse.com/docs/sql-reference/data-types/lowcardinality).
+ */
+const CH_MERGETREE = "https://clickhouse.com/docs/engines/table-engines/mergetree-family/mergetree";
+/**
+ * The rules. SQLRV001 to SQLRV005 are the shared ones; the rest are raised by
+ * a source reader for what only its dialect has, and live here so a report
+ * lists one catalog.
+ */
+export const REVIEW_RULES = {
+    SQLRV001: {
+        id: "SQLRV001",
+        name: "sort-key",
+        summary: "The ORDER BY key is a suggestion from the source's indexes.",
+        docs: [CH_MERGETREE],
+    },
+    SQLRV002: {
+        id: "SQLRV002",
+        name: "engine",
+        summary: "The table engine is a choice the source does not make.",
+        docs: [CH_MERGETREE, "https://clickhouse.com/docs/engines/table-engines/mergetree-family/replacingmergetree"],
+    },
+    SQLRV003: {
+        id: "SQLRV003",
+        name: "nullable",
+        summary: "A nullable column becomes Nullable(T), which costs storage and speed; a default can replace it.",
+        docs: ["https://clickhouse.com/docs/sql-reference/data-types/nullable"],
+    },
+    SQLRV004: {
+        id: "SQLRV004",
+        name: "low-cardinality",
+        summary: "A String column that may hold few distinct values could be LowCardinality(String).",
+        docs: ["https://clickhouse.com/docs/sql-reference/data-types/lowcardinality"],
+    },
+    SQLRV005: {
+        id: "SQLRV005",
+        name: "constraint",
+        summary: "ClickHouse does not enforce this constraint; it is kept as a comment.",
+        docs: ["https://clickhouse.com/docs/sql-reference/statements/create/table"],
+    },
+    SQLRV006: {
+        id: "SQLRV006",
+        name: "generated-column",
+        summary: "An identity, sequence, computed or system-generated column has no ClickHouse equivalent.",
+        docs: ["https://clickhouse.com/docs/sql-reference/statements/create/table"],
+    },
+    SQLRV007: {
+        id: "SQLRV007",
+        name: "type-judgment",
+        summary: "The type has no exact ClickHouse match; the nearest one was chosen.",
+        docs: ["https://clickhouse.com/docs/sql-reference/data-types"],
+    },
+    SQLRV008: {
+        id: "SQLRV008",
+        name: "default-expression",
+        summary: "A default expression was translated or dropped.",
+        docs: ["https://clickhouse.com/docs/sql-reference/statements/create/table"],
+    },
+    SQLRV009: {
+        id: "SQLRV009",
+        name: "storage-option",
+        summary: "A storage option of the source (filegroup, partitioning, compression, versioning) has no counterpart here.",
+        docs: [CH_MERGETREE],
+    },
+    SQLRV010: {
+        id: "SQLRV010",
+        name: "query-text",
+        summary: "A view body is copied as written and is not translated.",
+        docs: ["https://clickhouse.com/docs/sql-reference/statements/create/view"],
+    },
+    SQLRV011: {
+        id: "SQLRV011",
+        name: "not-carried",
+        summary: "An object or statement of the source is not carried over, and is named here.",
+        docs: [],
+    },
+    SQLRV012: {
+        id: "SQLRV012",
+        name: "emulator-rejected",
+        summary: "The ClickHouse emulator refused a statement the import would write.",
+        docs: [],
+    },
+};
+/** A finding as one line. */
+export function findingText(f) {
+    const where = f.column ? `${f.object}.${f.column}` : f.object;
+    const parts = [`${where}: ${f.message}`];
+    if (f.evidence.length > 0)
+        parts.push(`Evidence: ${f.evidence.join("; ")}.`);
+    if (f.suggestion)
+        parts.push(`Suggestion: ${f.suggestion}`);
+    return parts.join(" ");
+}
+const q = (name) => (/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) ? name : `\`${name.replace(/`/g, "\\`")}\``);
+const list = (names) => names.join(", ");
+const SORT_KEY_PRIORITY = ["explicit-order", "clustering-key", "primary-key", "unique-key"];
+export function sortKeyReview(input) {
+    const ranked = SORT_KEY_PRIORITY.flatMap((kind) => input.candidates.filter((c) => c.kind === kind && c.columns.length > 0));
+    const chosen = ranked[0];
+    const evidence = [];
+    const settings = {};
+    let orderBy = [];
+    let message;
+    let suggestion;
+    if (chosen) {
+        orderBy = [...chosen.columns];
+        evidence.push(`${chosen.source} (${list(chosen.columns)})`);
+        for (const other of ranked.slice(1))
+            evidence.push(`passed over: ${other.source} (${list(other.columns)})`);
+        if (chosen.descending && chosen.descending.length > 0)
+            evidence.push(`descending in the source, ascending here: ${list(chosen.descending)}`);
+        const nullable = orderBy.filter((c) => (input.columnTypes[c] ?? "").startsWith("Nullable("));
+        if (nullable.length > 0) {
+            settings.allow_nullable_key = "1";
+            evidence.push(`Nullable key column${nullable.length > 1 ? "s" : ""}: ${list(nullable)}, so allow_nullable_key = 1 is set`);
+        }
+        message = `ORDER BY (${list(orderBy.map(q))}) suggested from the ${chosen.kind.replace("-", " ")}.`;
+        suggestion = "Put the columns your queries filter on first, and the one with the fewest distinct values before the one with the most.";
+    }
+    else {
+        orderBy = [];
+        evidence.push("no clustering key, primary key or unique key in the source");
+        message = "ORDER BY tuple() used because the source gives no key to suggest.";
+        suggestion = "Choose the columns your queries filter on; a MergeTree table without a sort key cannot skip granules.";
+    }
+    return {
+        orderBy,
+        settings,
+        finding: { ruleId: "SQLRV001", severity: "warning", object: input.object, message, evidence, ...(suggestion ? { suggestion } : {}), ...(input.at ? { at: input.at } : {}) },
+    };
+}
+export const CDC_VERSION_COLUMN = { name: "_version", type: "UInt64" };
+export const CDC_DELETED_COLUMN = { name: "_deleted", type: "UInt8" };
+export function engineReview(input) {
+    const evidence = [...(input.facts ?? [])];
+    const unique = input.uniqueKey && input.uniqueKey.length > 0 ? list(input.uniqueKey) : undefined;
+    if (input.mode === "cdc") {
+        evidence.push(`the table is mirrored from a source that updates rows in place${unique ? `, keyed on (${unique})` : ""}`);
+        const version = input.cdcColumns?.version ?? CDC_VERSION_COLUMN.name;
+        const deleted = input.cdcColumns?.deleted ?? CDC_DELETED_COLUMN.name;
+        return {
+            engine: `ReplacingMergeTree(${version}, ${deleted})`,
+            extraColumns: [
+                { name: version, type: CDC_VERSION_COLUMN.type },
+                { name: deleted, type: CDC_DELETED_COLUMN.type },
+            ],
+            finding: {
+                ruleId: "SQLRV002",
+                severity: "warning",
+                object: input.object,
+                message: `ReplacingMergeTree(${version}, ${deleted}) chosen for a mirror; it adds the columns ${version} and ${deleted}.`,
+                evidence,
+                suggestion: `The sort key must be the source key, because rows with an equal sort key collapse to the highest ${version} on merge. Read with FINAL, or filter on ${deleted}, to see the collapsed rows before a merge.`,
+                ...(input.at ? { at: input.at } : {}),
+            },
+        };
+    }
+    if (unique)
+        evidence.push(`the source enforces uniqueness on (${unique}); MergeTree does not`);
+    return {
+        engine: "MergeTree",
+        extraColumns: [],
+        finding: {
+            ruleId: "SQLRV002",
+            severity: "warning",
+            object: input.object,
+            message: "MergeTree chosen; the source has no engine to carry over.",
+            evidence,
+            suggestion: unique
+                ? "If rows can arrive twice, ReplacingMergeTree collapses duplicates of the sort key on merge, not on insert."
+                : "Pick another MergeTree variant for pre-aggregation (SummingMergeTree, AggregatingMergeTree) when the table is a rollup.",
+            ...(input.at ? { at: input.at } : {}),
+        },
+    };
+}
+export function nullableReview(input) {
+    if (!input.sourceNullable)
+        return { type: input.type };
+    const evidence = [`${input.column} allows NULL in the source`];
+    let suggestion;
+    if (input.defaultExpr !== undefined) {
+        evidence.push(`it has the default ${input.defaultExpr}`);
+        suggestion = `If NULL is never stored on purpose, declare ${input.type} DEFAULT ${input.defaultExpr} and load the default instead of NULL.`;
+    }
+    else {
+        evidence.push("it has no default");
+        suggestion = `If a sentinel value can stand for missing, declare ${input.type} DEFAULT <sentinel>; a Nullable column keeps a separate null mask.`;
+    }
+    return {
+        type: `Nullable(${input.type})`,
+        finding: {
+            ruleId: "SQLRV003",
+            severity: "warning",
+            object: input.object,
+            column: input.column,
+            message: `Kept as Nullable(${input.type}) so no stored NULL is lost.`,
+            evidence,
+            suggestion,
+            ...(input.at ? { at: input.at } : {}),
+        },
+    };
+}
+// --- LowCardinality -------------------------------------------------------
+/** Last words of a column name that usually mean a small set of values. */
+const LOW_CARDINALITY_WORDS = new Set("status state type kind category country currency region level channel gender lang language locale unit code flag method mode grade tier priority".split(" "));
+/** A column name's last word: `order_status` and `OrderStatus` both give `status`. */
+function lastWord(name) {
+    const words = name.replace(/([a-z0-9])([A-Z])/g, "$1_$2").split(/[^A-Za-z0-9]+/).filter(Boolean);
+    return (words[words.length - 1] ?? "").toLowerCase();
+}
+/** The longest declared length that still counts as supporting evidence. */
+export const LOW_CARDINALITY_MAX_LENGTH = 50;
+export function lowCardinalityReview(input) {
+    if (input.type !== "String" && input.type !== "Nullable(String)")
+        return undefined;
+    if (!LOW_CARDINALITY_WORDS.has(lastWord(input.column)))
+        return undefined;
+    const evidence = [`the name ${input.column} ends in a word that usually names a small set of values`, `the type is ${input.type}`];
+    if (input.maxLength !== undefined && input.maxLength <= LOW_CARDINALITY_MAX_LENGTH)
+        evidence.push(`the declared length is ${String(input.maxLength)}`);
+    evidence.push("no data was read, so the distinct count is unknown");
+    const wrapped = input.type.startsWith("Nullable(") ? "LowCardinality(Nullable(String))" : "LowCardinality(String)";
+    return {
+        ruleId: "SQLRV004",
+        severity: "warning",
+        object: input.object,
+        column: input.column,
+        message: `${input.column} is a LowCardinality candidate and stays ${input.type}.`,
+        evidence,
+        suggestion: `ClickHouse documents ${wrapped} as helpful below roughly ten thousand distinct values; count them with uniqExact(${q(input.column)}) before changing the type.`,
+        ...(input.at ? { at: input.at } : {}),
+    };
+}
+const oneLine = (s) => s.replace(/\s+/g, " ").trim();
+export function constraintReview(input) {
+    const comments = [];
+    const findings = [];
+    for (const c of input.constraints) {
+        const label = c.name ? ` ${c.name}` : "";
+        let text;
+        let message;
+        let suggestion;
+        if (c.kind === "foreign-key") {
+            const target = c.references ? `${c.references.table} (${list(c.references.columns)})` : "an unknown table";
+            text = `FOREIGN KEY${label} (${list(c.columns ?? [])}) REFERENCES ${target}`;
+            message = `Foreign key${label} is not enforced by ClickHouse and is kept as a comment.`;
+            suggestion = "Check referential integrity in the load pipeline, or join on the column and accept orphan rows.";
+        }
+        else if (c.kind === "unique") {
+            text = `UNIQUE${label} (${list(c.columns ?? [])})`;
+            message = `Unique constraint${label} is not enforced by ClickHouse and is kept as a comment.`;
+            suggestion = "A MergeTree sort key does not make rows unique. ReplacingMergeTree collapses duplicates on merge only.";
+        }
+        else if (c.kind === "exclusion") {
+            text = `EXCLUDE${label} ${oneLine(c.expression ?? "")}`;
+            message = `Exclusion constraint${label} is not enforced by ClickHouse and is kept as a comment.`;
+            suggestion = "ClickHouse has no constraint that rejects overlapping rows. Check for overlaps in the load, or in a query.";
+        }
+        else {
+            text = `CHECK${label} ${oneLine(c.expression ?? "")}`;
+            message = `Check constraint${label} is not translated and is kept as a comment.`;
+            suggestion = "ClickHouse has CONSTRAINT name CHECK expr, enforced on INSERT. Rewrite the expression in ClickHouse SQL to use it.";
+        }
+        comments.push(`-- ${oneLine(text)}`);
+        findings.push({
+            ruleId: "SQLRV005",
+            severity: "warning",
+            object: input.object,
+            ...(c.kind !== "check" && c.kind !== "exclusion" && c.columns && c.columns.length === 1 ? { column: c.columns[0] } : {}),
+            message,
+            evidence: [oneLine(text)],
+            suggestion,
+            ...(c.at ? { at: c.at } : {}),
+        });
+    }
+    return { comments, findings };
+}

package/dist/codegen/classifier-page.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"classifier-page.d.ts","sourceRoot":"","sources":["../../src/codegen/classifier-page.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAcH,wBAAgB,oBAAoB,IAAI,MAAM,CAiF7C"}
+{"version":3,"file":"classifier-page.d.ts","sourceRoot":"","sources":["../../src/codegen/classifier-page.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAeH,wBAAgB,oBAAoB,IAAI,MAAM,CAiG7C"}

package/dist/codegen/classifier-page.js

@@ -8,6 +8,7 @@ const LABEL = {
     drop: "drop",
     metadata: "metadata only",
     rewrite: "background rewrite",
+    replace: "replace table",
     rebuild: "rebuild",
 };
 const cell = (s) => s.replace(/\|/g, "\\|");
@@ -16,7 +17,7 @@ export function renderClassifierPage() {
     return `---
 title: "Planning ClickHouse Changes"
 label: "Planning and the Change Classifier"
-description: "How a ClickHouse schema change is classified as metadata only, a background rewrite or a rebuild, each with the ALTER restriction behind it"
+description: "How a ClickHouse schema change is classified as metadata only, a background rewrite, a replacement or a rebuild, each with the ALTER restriction behind it"
 diataxis: reference
 group: "ClickHouse"
 order: 1
@@ -30,9 +31,10 @@ Every change between two ClickHouse schemas is classified before anything runs:
 |---|---|
 | metadata only | the server records the change and touches no existing data |
 | background rewrite | a mutation rewrites existing parts in the background, with no rollback |
+| replace table | ClickHouse cannot alter a stream-engine table, and it stores no rows, so the applier drops and creates it around the views that read it |
 | rebuild | ClickHouse cannot make the change to the existing table, and the data has to be copied into a new one; a plan refuses to make it in place |
 
-Two commands report the classification, and \`chant lifecycle plan\` reports it as each update's disruption (metadata only is \`in-place\`, a rewrite \`rolling\`, a rebuild \`replace\`):
+Two commands report the classification, and \`chant lifecycle plan\` reports it as each update's disruption (metadata only is \`in-place\`, a rewrite \`rolling\`, a replacement or a rebuild \`replace\`):
 
 \`\`\`bash
 chant sql diff base.json head.json     # two chant build outputs, offline
@@ -41,13 +43,28 @@ chant sql plan prod schema.json        # a build output against the prod server
 
 Both exit 2 when a change needs a rebuild. \`chant sql diff\` compares a pull request's base and head builds with no server. \`chant sql plan\` reads the server \`sql.profiles.<env>\` binds, and asks that server's formatter about any expression the normalization rules leave different, so the server's own rewriting (\`INTERVAL 1 DAY\` as \`toIntervalDay(1)\`, \`a+b*2\` as \`a + (b * 2)\`) is not reported as a change.
 
-\`chant sql plan\` also exits 2 when a declared database holds an object whose definition it cannot read. It names each one under \`Refused\` (\`unreadable\` in \`--json\`) instead of leaving it out. Dictionaries are read and planned like the other objects: any change but a comment replaces one (SQLCH245).
+\`chant sql plan\` also exits 2 when a declared database holds an object whose definition it cannot read. It names each one under \`Refused\` (\`unreadable\` in \`--json\`) instead of leaving it out. It exits 2 as well when the build names a named collection the server doesn't have, naming each under \`Refused\` (\`namedCollections.missing\` in \`--json\`); see [Named collections](../clickhouse-ddl/#named-collections). Dictionaries are read and planned like the other objects: any change but a comment replaces one (SQLCH245).
 
 A clause chant's parser does not know ([Plain .sql Files](../sql-files/#clauses-chant-does-not-read)) is compared as text, and the rest of its object is planned as usual. A change to one is SQLCH290, refused for review with no rebuild Op suggested, since chant cannot tell what the change takes. A clause only the server prints is a hint, not a change: a newer server printing a clause with its default looks the same as a clause removed from the declaration.
 
+## Stream tables
+
+A table on Kafka, S3Queue, AzureQueue, RabbitMQ, NATS or FileLog stores no rows. It is the consumer that a materialized view reads through, and the position it has reached lives in the broker's committed offsets or in Keeper. There is no \`ALTER\` for its engine arguments, settings or columns, and there is nothing to copy, so a change to one is neither metadata nor a rebuild. It is SQLCH225, in the replace class (\`diffStreamTable\` in \`src/clickhouse/plan/diff.ts\`). The plan refuses nothing for it.
+
+The applier makes the change in place, in this order (\`replaceStreamSteps\` in \`src/clickhouse/apply/statements.ts\`): \`DETACH TABLE\` for each materialized view on the server that reads the table, \`DROP TABLE ... SYNC\`, the declared \`CREATE TABLE\`, then \`ATTACH TABLE\` for the same views. The plan names the views in a hint. Messages wait in the broker, and files wait in the bucket, while the views are detached. A change that moves a table to or from a stream engine is still SQLCH223, a rebuild, and \`ClickHouseRebuildOp\` refuses a stream-engine table by name (\`streamRefusal\` in \`src/clickhouse/rebuild/observe.ts\`).
+
+Two replacements lose the table's position, and the change carries a note saying so (\`positionWarnings\` in \`src/clickhouse/plan/stream.ts\`):
+
+- A different \`kafka_group_name\`, as a setting or as the third engine argument, is a different consumer group. It starts from its own committed offsets, or from the start of the topic when it has none.
+- A different S3Queue \`keeper_path\` is a different record of processed files. Every file already seen is processed again.
+
+An apply killed between the detach and the attach leaves the views detached, and a detached view is not in \`system.tables\`. Nothing re-attaches it: run \`ATTACH TABLE\` for each view by hand, then plan again. This has not been run against a server.
+
+The \`StreamIngest\` composite declares the stream table, the view and the MergeTree target together: see [Composites](../composites/#streamingest).
+
 ## After the plan
 
-The plan only reports. [\`clickhouseApply\`](../applying/) makes the metadata-only and background-rewrite changes, and refuses a rebuild the same way the plan does, sending nothing for that object. A rebuild runs as \`ClickHouseRebuildOp\`, a gated migration Op that creates the new table, backfills it, verifies it and swaps it in: see [Rebuilding a Table](../rebuild/). Both commands print the Op declaration to start from for each refused table.
+The plan only reports. [\`clickhouseApply\`](../applying/) makes the metadata-only, background-rewrite and stream-table changes, and refuses a rebuild the same way the plan does, sending nothing for that object. A rebuild runs as \`ClickHouseRebuildOp\`, a gated migration Op that creates the new table, backfills it, verifies it and swaps it in: see [Rebuilding a Table](../rebuild/). Both commands print the Op declaration to start from for each refused table.
 
 \`chant migrate\` plays no part in any of this. It translates a file from one lexicon's format into another's, such as a GitHub Actions workflow into GitLab CI, and does not run schema migrations.
 

package/dist/codegen/clickhouse-module.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"clickhouse-module.d.ts","sourceRoot":"","sources":["../../src/codegen/clickhouse-module.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAc,MAAM,iBAAiB,CAAC;AA6BrE,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAElD;AAmED,MAAM,WAAW,cAAc;IAC7B,0GAA0G;IAC1G,YAAY,EAAE,MAAM,CAAC;IACrB,0EAA0E;IAC1E,MAAM,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,iBAAiB,GAAG,cAAc,CAmKjF"}
+{"version":3,"file":"clickhouse-module.d.ts","sourceRoot":"","sources":["../../src/codegen/clickhouse-module.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAc,MAAM,iBAAiB,CAAC;AA6BrE,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAElD;AAmED,MAAM,WAAW,cAAc;IAC7B,0GAA0G;IAC1G,YAAY,EAAE,MAAM,CAAC;IACrB,0EAA0E;IAC1E,MAAM,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,iBAAiB,GAAG,cAAc,CAsKjF"}

package/dist/codegen/clickhouse-module.js

@@ -236,6 +236,9 @@ export function renderClickHouseModule(catalog) {
         `/** The table functions (\`system.table_functions\`): \`numbers\`, \`remote\`, \`s3\`. */`,
         `export const TABLE_FUNCTIONS: readonly string[] = ${JSON.stringify(catalog.tableFunctions)};`,
         "",
+        `/** The input and output formats (\`system.formats\`): \`Parquet\`, \`JSONEachRow\`. */`,
+        `export const FORMATS: readonly string[] = ${JSON.stringify(catalog.formats.map((f) => f.name))};`,
+        "",
         `export const SKIP_INDEX_TYPES: Readonly<Record<SkipIndexType, SkipIndexSpec>> = {`,
         ...indexLines,
         "};",

package/dist/codegen/coverage-page.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"coverage-page.d.ts","sourceRoot":"","sources":["../../src/codegen/coverage-page.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AA4BH,wBAAgB,kBAAkB,IAAI,MAAM,CA4C3C"}
+{"version":3,"file":"coverage-page.d.ts","sourceRoot":"","sources":["../../src/codegen/coverage-page.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AA6BH,wBAAgB,kBAAkB,IAAI,MAAM,CA6C3C"}

package/dist/codegen/coverage-page.js

@@ -5,7 +5,7 @@
  */
 import { KIND_COVERAGE, PINNED_COMMANDS } from "../coverage.js";
 const cell = (s) => s.replace(/\|/g, "\\|").replace(/[{}<>]/g, (c) => `\\${c}`);
-const STATUS = { declared: "declared", refused: "refused", "out-of-scope": "out of scope" };
+const STATUS = { declared: "declared", referenced: "referenced", refused: "refused", "out-of-scope": "out of scope" };
 function table(dialect) {
     const rows = [...KIND_COVERAGE[dialect]].sort((a, b) => (a.command < b.command ? -1 : a.command > b.command ? 1 : 0));
     return [
@@ -21,7 +21,8 @@ function table(dialect) {
 function counts(dialect) {
     const rows = KIND_COVERAGE[dialect];
     const n = (s) => rows.filter((r) => r.status === s).length;
-    return `${n("declared")} declared, ${n("refused")} refused, ${n("out-of-scope")} out of scope`;
+    const referenced = n("referenced") > 0 ? `, ${n("referenced")} referenced` : "";
+    return `${n("declared")} declared${referenced}, ${n("refused")} refused, ${n("out-of-scope")} out of scope`;
 }
 export function renderCoveragePage() {
     const pg = PINNED_COMMANDS.postgres.map((p) => p.version).join(", ");
@@ -29,7 +30,7 @@ export function renderCoveragePage() {
     return `---
 title: "Object Coverage"
 label: "Object Coverage"
-description: "Every kind of object each pinned Postgres and ClickHouse server can create, marked declared, refused with a reason, or out of scope, and what import does with each"
+description: "Every kind of object each pinned Postgres and ClickHouse server can create, marked declared, referenced, refused with a reason, or out of scope, and what import does with each"
 diataxis: reference
 order: 3
 ---
@@ -39,6 +40,7 @@ order: 3
 Each row is a \`CREATE\` command a pinned server has, or one form of a command where chant treats the forms apart (\`CREATE TYPE ... AS ENUM\` is declared, \`CREATE TYPE ... AS RANGE\` is not). The commands come from the servers' own sources at each pinned version (\`scripts/sql-create-commands.ts\`), and a test fails when a pinned server has a command with no row here, so a new server release cannot add a kind nobody looked at.
 
 - **declared**: a tag declares it, and import writes it as that declaration.
+- **referenced**: the environment's, and never declared: a declaration names one, and chant never reads its values. A ClickHouse named collection is one ([#4395](https://github.com/INTENTIUS/chant/issues/4395)): \`ENGINE = S3(lake_creds, format = 'Parquet')\` names \`lake_creds\`, the build records the name, and \`chant sql plan\` fails when the server has no collection of that name. \`chant import --from <env>\` lists the server's SQL-defined collections as prerequisites and imports the rest. See [Named collections](../clickhouse-ddl/#named-collections).
 - **refused**: no declaration holds it yet. \`chant import --from <env>\` fails when the server has one, naming every such object (kind, schema, name) at once, and \`chant import schema.sql\` fails on the statement. Nothing is left out without a word.
 - **out of scope**: the environment's, not a schema's: import never reads it.
 

package/dist/codegen/docs.js

@@ -120,7 +120,7 @@ Postgres:
 - [Getting Started with Postgres](./postgres-getting-started/): declare a
   schema, apply it to a local server, add a column and an index, and see a
   rename refused.
-- [Declaring Postgres Objects](./postgres-ddl/): the eight tags,
+- [Declaring Postgres Objects](./postgres-ddl/): the tags for every schema-level kind,
   constraints, \`nextval(\${seq})\`, \`COMMENT ON\` and \`literal()\`.
 - [Importing a Postgres Server](./postgres-importing/): canonical text from
   the catalog's printers, and the objects that belong to someone else.

package/dist/codegen/postgres-classifier-page.js

@@ -67,7 +67,7 @@ Two rules change class across the supported majors: a STORED generated column's
 
 ## Identity and renames
 
-An object is identified by its export name between two builds, and by its schema-qualified name against a server, which has no export names. Tables, views, materialized views, sequences and indexes share one namespace per schema, types and domains another, and schemas and extensions are matched by name. Functions, procedures and aggregates share a namespace of their own and are told apart by their input parameter types, so an overload is another object; composite and range types share the types' namespace, collations have one of their own, a cast is named by its two types and an event trigger by its name in the database; a trigger is matched by its name on its table.
+An object is identified by its export name between two builds, and by its schema-qualified name against a server, which has no export names. Tables, views, materialized views, sequences and indexes share one namespace per schema, types and domains another, and schemas and extensions are matched by name. Functions, procedures and aggregates share a namespace of their own and are told apart by their input parameter types, so an overload is another object; composite and range types share the types' namespace, collations have one of their own, extended statistics another, an operator by its symbol and operand types, a cast by its two types and an event trigger, a foreign data wrapper, a foreign server and a publication by their names in the database; a trigger is matched by its name on its table.
 
 A rename is declared where it happens. Before a CREATE, \`-- previously: <old name>\` says the object had another name; on a column's line, it says the column did:
 

package/dist/codegen/registry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/codegen/registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AA0CH,wBAAgB,aAAa,IAAI,MAAM,CAKtC"}
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/codegen/registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AA+CH,wBAAgB,aAAa,IAAI,MAAM,CAKtC"}

package/dist/codegen/registry.js

@@ -37,6 +37,11 @@ const KINDS = {
     PostgresCollation: POSTGRES_ENTITY_TYPES.collation,
     PostgresCast: POSTGRES_ENTITY_TYPES.cast,
     PostgresEventTrigger: POSTGRES_ENTITY_TYPES.eventTrigger,
+    PostgresOperator: POSTGRES_ENTITY_TYPES.operator,
+    PostgresStatistics: POSTGRES_ENTITY_TYPES.statistics,
+    PostgresForeignDataWrapper: POSTGRES_ENTITY_TYPES.foreignDataWrapper,
+    PostgresForeignServer: POSTGRES_ENTITY_TYPES.foreignServer,
+    PostgresPublication: POSTGRES_ENTITY_TYPES.publication,
     PostgresExtension: POSTGRES_ENTITY_TYPES.extension,
     PostgresFunction: POSTGRES_ENTITY_TYPES.function,
     PostgresProcedure: POSTGRES_ENTITY_TYPES.procedure,

package/dist/composites/catalog.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"catalog.d.ts","sourceRoot":"","sources":["../../src/composites/catalog.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAE/D,eAAO,MAAM,gBAAgB,EAAE,cAAc,EA2gB5C,CAAC"}
+{"version":3,"file":"catalog.d.ts","sourceRoot":"","sources":["../../src/composites/catalog.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAE/D,eAAO,MAAM,gBAAgB,EAAE,cAAc,EA+mB5C,CAAC"}

package/dist/composites/catalog.js

@@ -210,6 +210,41 @@ export const compositeCatalog = [
             }
         ]
     },
+    {
+        "name": "OtelTables",
+        "lexicon": "sql",
+        "description": "The logs, traces and metrics tables the OpenTelemetry Collector's clickhouse exporter writes, as it creates them at v0.130.0.",
+        "bundles": [
+            "ClickHouseTable",
+            "ClickHouseView"
+        ],
+        "params": [
+            {
+                "name": "database",
+                "type": "string",
+                "required": false,
+                "description": "The database the tables are created in, the exporter's `database`."
+            },
+            {
+                "name": "ttl",
+                "type": "string",
+                "required": false,
+                "description": "How long rows are kept, as the exporter's `ttl` takes it: a Go duration such as `72h` or `30m`."
+            },
+            {
+                "name": "tables",
+                "type": "OtelTableNames",
+                "required": false,
+                "description": "Table names other than the exporter's defaults, as set on the exporter."
+            },
+            {
+                "name": "traceIdLookup",
+                "type": "boolean",
+                "required": false,
+                "description": "Declare the trace-id lookup table and its materialized view (default true)."
+            }
+        ]
+    },
     {
         "name": "RefreshedView",
         "lexicon": "sql",
@@ -468,6 +503,71 @@ export const compositeCatalog = [
             }
         ]
     },
+    {
+        "name": "StreamIngest",
+        "lexicon": "sql",
+        "description": "A stream table, the materialized view that reads it, and the MergeTree target the view writes.",
+        "bundles": [
+            "ClickHouseTable",
+            "ClickHouseView"
+        ],
+        "params": [
+            {
+                "name": "name",
+                "type": "string",
+                "required": true,
+                "description": "The target table's name."
+            },
+            {
+                "name": "columns",
+                "type": "string",
+                "required": true,
+                "description": "The target's columns, as SQL: `ts DateTime, user_id UInt64, kind LowCardinality(String)`."
+            },
+            {
+                "name": "engine",
+                "type": "string",
+                "required": true,
+                "description": "The stream table's engine, with a named collection as its first argument rather than inline credentials: `Kafka(kafka_creds)`, `S3Queue(lake_creds)`."
+            },
+            {
+                "name": "settings",
+                "type": "string",
+                "required": false,
+                "description": "The engine's settings, the body of a `SETTINGS` clause: `kafka_topic_list = 'events', kafka_group_name = 'ingest', kafka_format = 'JSONEachRow'`."
+            },
+            {
+                "name": "streamColumns",
+                "type": "string",
+                "required": false,
+                "description": "The stream table's columns when they differ from the target's: the message as it arrives."
+            },
+            {
+                "name": "select",
+                "type": "string",
+                "required": false,
+                "description": "The view's select list over the stream table, each item aliased to a target column."
+            },
+            {
+                "name": "orderBy",
+                "type": "string",
+                "required": true,
+                "description": "The target's sort key (a column list or an expression)."
+            },
+            {
+                "name": "targetEngine",
+                "type": "string",
+                "required": false,
+                "description": "The target's engine (default `MergeTree`)."
+            },
+            {
+                "name": "partitionBy",
+                "type": "string",
+                "required": false,
+                "description": "The target's partition key."
+            }
+        ]
+    },
     {
         "name": "TenantTable",
         "lexicon": "sql",

package/dist/composites/clickhouse.d.ts

@@ -4,4 +4,6 @@ export { EventsTable, type EventsTableProps, type EventsTableMembers } from "./e
 export { RollupView, type RollupViewProps, type RollupViewMembers } from "./rollup-view.js";
 export { CdcMirror, type CdcMirrorProps, type CdcMirrorMembers } from "./cdc-mirror.js";
 export { ShardedTable, type ShardedTableProps, type ShardedTableMembers } from "./sharded-table.js";
+export { OtelTables, OTEL_EXPORTER_DEFAULT_TABLES, otelTtlClause, type OtelTablesProps, type OtelTablesMembers, type OtelTableNames } from "./otel-tables.js";
+export { StreamIngest, type StreamIngestProps, type StreamIngestMembers } from "./stream-ingest.js";
 //# sourceMappingURL=clickhouse.d.ts.map

package/dist/composites/clickhouse.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"clickhouse.d.ts","sourceRoot":"","sources":["../../src/composites/clickhouse.ts"],"names":[],"mappings":"AAAA,wFAAwF;AAExF,OAAO,EAAE,cAAc,EAAE,KAAK,mBAAmB,EAAE,KAAK,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AACzG,OAAO,EAAE,WAAW,EAAE,KAAK,gBAAgB,EAAE,KAAK,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAC7F,OAAO,EAAE,UAAU,EAAE,KAAK,eAAe,EAAE,KAAK,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACzF,OAAO,EAAE,SAAS,EAAE,KAAK,cAAc,EAAE,KAAK,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrF,OAAO,EAAE,YAAY,EAAE,KAAK,iBAAiB,EAAE,KAAK,mBAAmB,EAAE,MAAM,iBAAiB,CAAC"}
+{"version":3,"file":"clickhouse.d.ts","sourceRoot":"","sources":["../../src/composites/clickhouse.ts"],"names":[],"mappings":"AAAA,wFAAwF;AAExF,OAAO,EAAE,cAAc,EAAE,KAAK,mBAAmB,EAAE,KAAK,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AACzG,OAAO,EAAE,WAAW,EAAE,KAAK,gBAAgB,EAAE,KAAK,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAC7F,OAAO,EAAE,UAAU,EAAE,KAAK,eAAe,EAAE,KAAK,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACzF,OAAO,EAAE,SAAS,EAAE,KAAK,cAAc,EAAE,KAAK,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrF,OAAO,EAAE,YAAY,EAAE,KAAK,iBAAiB,EAAE,KAAK,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACjG,OAAO,EAAE,UAAU,EAAE,4BAA4B,EAAE,aAAa,EAAE,KAAK,eAAe,EAAE,KAAK,iBAAiB,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAC3J,OAAO,EAAE,YAAY,EAAE,KAAK,iBAAiB,EAAE,KAAK,mBAAmB,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/composites/clickhouse.js

@@ -4,3 +4,5 @@ export { EventsTable } from "./events-table.js";
 export { RollupView } from "./rollup-view.js";
 export { CdcMirror } from "./cdc-mirror.js";
 export { ShardedTable } from "./sharded-table.js";
+export { OtelTables, OTEL_EXPORTER_DEFAULT_TABLES, otelTtlClause } from "./otel-tables.js";
+export { StreamIngest } from "./stream-ingest.js";