@intentius/chant-lexicon-sql 0.124.31 → 0.124.33

package/dist/codegen/clickhouse-module.js

@@ -0,0 +1,249 @@
+/**
+ * Render `src/generated/clickhouse.ts`, the ClickHouse dialect's generated
+ * surface: string-literal unions for every named thing in the catalog, the two
+ * settings interfaces, and the tables lint and the LSP read (engines with
+ * their typed arguments, type families, codecs, skip index types, settings).
+ *
+ * The overlays are merged here, and checked against the catalog in both
+ * directions. An overlay key the pinned server does not have, or a codec at
+ * the pin with no overlay entry, is an error: a pin move that changes the
+ * grammar has to change the overlay in the same commit.
+ */
+import { parseEngineSyntax } from "../clickhouse/engine-syntax.js";
+import { ENGINE_ARGUMENTS } from "../clickhouse/overlays/engines.js";
+import { SKIP_INDEX_PARAMETERS } from "../clickhouse/overlays/skip-indexes.js";
+import { CODEC_OVERLAY } from "../clickhouse/overlays/codecs.js";
+import { TYPE_PARAMETERS } from "../clickhouse/overlays/types.js";
+import { TTL_SETTINGS } from "../clickhouse/overlays/ttl.js";
+import { PROJECTION_SETTINGS } from "../clickhouse/overlays/projections.js";
+/** Setting types with a TypeScript form; every enum-typed setting (no member list in the catalog) is `string`. */
+const SETTING_TS_TYPES = {
+    Bool: "boolean",
+    BoolAuto: 'boolean | "auto"',
+    UInt64: "number",
+    Int64: "number",
+    UInt32: "number",
+    Int32: "number",
+    Float: "number",
+    Double: "number",
+    Seconds: "number",
+    Milliseconds: "number",
+    NonZeroUInt64: "number",
+    MaxThreads: 'number | "auto"',
+    UInt64Auto: 'number | "auto"',
+    FloatAuto: 'number | "auto"',
+};
+export function settingTsType(type) {
+    return SETTING_TS_TYPES[type] ?? "string";
+}
+const lit = (names) => names.length > 0 ? names.map((n) => JSON.stringify(n)).join(" | ") : "never";
+const key = (name) => (/^[A-Za-z_$][\w$]*$/.test(name) ? name : JSON.stringify(name));
+/** A JSDoc line cannot contain `*\/`. */
+const doc = (text) => text.replace(/\*\//g, "*\\/");
+function mergeArguments(owner, parsed, overlay, errors) {
+    if (!parsed) {
+        if (overlay)
+            errors.push(`${owner}: the overlay types arguments, but the syntax line is not readable`);
+        return { typed: false };
+    }
+    const names = new Set(parsed.args.map((a) => a.name));
+    for (const name of Object.keys(overlay ?? {})) {
+        if (!names.has(name))
+            errors.push(`${owner}: overlay argument "${name}" is not in the syntax line at this pin`);
+    }
+    const args = parsed.args.map((a) => {
+        const o = overlay?.[a.name];
+        return {
+            name: a.name,
+            position: a.position,
+            optional: o?.optional ?? a.optional,
+            repeated: a.repeated,
+            named: a.named,
+            kind: o?.kind ?? (a.quoted ? "string" : "unknown"),
+            ...(o?.values ? { values: o.values } : {}),
+            ...(o?.range ? { range: o.range } : {}),
+            ...(o?.note ? { note: o.note } : {}),
+        };
+    });
+    return { args, typed: args.every((a) => a.kind !== "unknown") };
+}
+function settingsInterface(name, rows) {
+    const lines = [`export interface ${name} {`];
+    for (const r of rows) {
+        const tags = [doc(r.summary)];
+        if (r.obsolete || r.tier === "Obsolete")
+            tags.push("@deprecated obsolete at this pin");
+        if (r.tier === "Experimental" || r.tier === "Beta")
+            tags.push(`@tier ${r.tier}`);
+        if (r.aliasFor)
+            tags.push(`@alias ${r.aliasFor}`);
+        tags.push(`@default ${doc(JSON.stringify(r.default))}`);
+        lines.push(`  /** ${tags.filter(Boolean).join(" | ")} */`);
+        lines.push(`  ${key(r.name)}?: ${settingTsType(r.type)};`);
+    }
+    lines.push("}");
+    return lines.join("\n");
+}
+function settingsTable(name, keyType, rows) {
+    const lines = [`export const ${name}: Readonly<Record<${keyType}, SettingSpec>> = {`];
+    for (const r of rows) {
+        lines.push(`  ${key(r.name)}: ${JSON.stringify({ type: r.type, default: r.default, tier: r.tier, obsolete: r.obsolete, readonly: r.readonly })},`);
+    }
+    lines.push("};");
+    return lines.join("\n");
+}
+export function renderClickHouseModule(catalog) {
+    const errors = [];
+    const notes = [];
+    // ── Cross-checks of the overlays against the catalog ──────────────
+    const engineNames = new Set(catalog.tableEngines.map((e) => e.name));
+    for (const name of Object.keys(ENGINE_ARGUMENTS)) {
+        if (!engineNames.has(name))
+            errors.push(`overlays/engines.ts: engine ${name} is not in the catalog at ${catalog.version}`);
+    }
+    const indexNames = new Set(catalog.skipIndexTypes.map((e) => e.name));
+    for (const name of Object.keys(SKIP_INDEX_PARAMETERS)) {
+        if (!indexNames.has(name))
+            errors.push(`overlays/skip-indexes.ts: index type ${name} is not in the catalog`);
+    }
+    const codecNames = new Set(catalog.codecs.map((c) => c.name));
+    for (const name of Object.keys(CODEC_OVERLAY)) {
+        if (!codecNames.has(name))
+            errors.push(`overlays/codecs.ts: codec ${name} is not in the catalog`);
+    }
+    for (const name of codecNames) {
+        if (!CODEC_OVERLAY[name])
+            errors.push(`overlays/codecs.ts: codec ${name} at ${catalog.version} has no entry`);
+    }
+    const families = new Set(catalog.typeFamilies.filter((t) => !t.aliasOf).map((t) => t.name));
+    for (const name of Object.keys(TYPE_PARAMETERS)) {
+        if (!families.has(name))
+            errors.push(`overlays/types.ts: type family ${name} is not in the catalog`);
+    }
+    const mtSettings = new Set(catalog.mergeTreeSettings.map((s) => s.name));
+    for (const name of [...TTL_SETTINGS, ...PROJECTION_SETTINGS]) {
+        if (!mtSettings.has(name))
+            errors.push(`overlays: MergeTree setting ${name} is not in the catalog`);
+    }
+    // ── Engines ────────────────────────────────────────────────────────
+    const engineLines = [];
+    for (const e of catalog.tableEngines) {
+        const parsed = parseEngineSyntax(e.syntax);
+        const { args, typed } = mergeArguments(`engine ${e.name}`, parsed, ENGINE_ARGUMENTS[e.name], errors);
+        const mergeTree = /MergeTree$/.test(e.name);
+        const replicates = /^Replicated.+MergeTree$/.test(e.name) ? e.name.slice("Replicated".length) : undefined;
+        if (mergeTree && !typed)
+            notes.push(`engine ${e.name}: arguments not fully typed`);
+        const spec = {
+            capabilities: e.capabilities,
+            syntax: e.syntax,
+            ...(args ? { args } : {}),
+            typed,
+            mergeTree,
+            ...(replicates ? { replicates } : {}),
+            summary: e.summary,
+        };
+        engineLines.push(`  ${key(e.name)}: ${JSON.stringify(spec)},`);
+    }
+    const indexLines = [];
+    for (const t of catalog.skipIndexTypes) {
+        const { args } = mergeArguments(`index ${t.name}`, parseEngineSyntax(t.syntax), SKIP_INDEX_PARAMETERS[t.name], errors);
+        indexLines.push(`  ${key(t.name)}: ${JSON.stringify({ syntax: t.syntax, ...(args ? { args } : {}), summary: t.summary })},`);
+    }
+    if (errors.length > 0) {
+        throw new Error(`The ClickHouse overlays disagree with the catalog at ${catalog.version}:\n  ${errors.join("\n  ")}\n` +
+            "Update src/clickhouse/overlays/ for this pin.");
+    }
+    const canonical = catalog.typeFamilies.filter((t) => !t.aliasOf);
+    const aliases = catalog.typeFamilies.filter((t) => t.aliasOf);
+    const mergeTreeEngines = catalog.tableEngines.filter((e) => /MergeTree$/.test(e.name));
+    const header = (what) => [
+        `// Generated by \`npm run generate\` from the ClickHouse catalog at ${catalog.version}`,
+        `// (src/spec/clickhouse-catalog.snapshot.json) and the overlays in src/clickhouse/overlays/. Do not edit.`,
+        `// ${what}`,
+        "",
+    ];
+    const declarations = [
+        ...header("The type surface: unions of every named thing in the catalog, and the two settings interfaces."),
+        `/** The ClickHouse release these types were read from. */`,
+        `export const CLICKHOUSE_VERSION = ${JSON.stringify(catalog.version)};`,
+        "",
+        `export type TableEngineName = ${lit(catalog.tableEngines.map((e) => e.name))};`,
+        `export type MergeTreeEngineName = ${lit(mergeTreeEngines.map((e) => e.name))};`,
+        `export type DatabaseEngineName = ${lit(catalog.databaseEngines.map((e) => e.name))};`,
+        `export type ColumnTypeFamily = ${lit(canonical.map((t) => t.name))};`,
+        `export type ColumnTypeAlias = ${lit(aliases.map((t) => t.name))};`,
+        `export type ColumnTypeName = ColumnTypeFamily | ColumnTypeAlias;`,
+        `export type CodecName = ${lit(catalog.codecs.map((c) => c.name))};`,
+        `export type SkipIndexType = ${lit(catalog.skipIndexTypes.map((c) => c.name))};`,
+        `export type AggregateCombinator = ${lit(catalog.aggregateCombinators.filter((c) => !c.internal).map((c) => c.name))};`,
+        `export type TableFunctionName = ${lit(catalog.tableFunctions)};`,
+        `export type FormatName = ${lit(catalog.formats.map((f) => f.name))};`,
+        `export type InputFormatName = ${lit(catalog.formats.filter((f) => f.input).map((f) => f.name))};`,
+        `export type OutputFormatName = ${lit(catalog.formats.filter((f) => f.output).map((f) => f.name))};`,
+        `export type AggregateFunctionName = ${lit(catalog.functions.filter((f) => f.aggregate).map((f) => f.name))};`,
+        `export type ScalarFunctionName = ${lit(catalog.functions.filter((f) => !f.aggregate).map((f) => f.name))};`,
+        "",
+        settingsInterface("MergeTreeSettings", catalog.mergeTreeSettings),
+        "",
+        settingsInterface("QuerySettings", catalog.querySettings),
+        "",
+    ].join("\n");
+    const tables = [
+        ...header("The tables lint and the LSP read, typed by ./clickhouse-types."),
+        `import type { CodecSpec, DatabaseEngineSpec, FunctionSpec, SettingSpec, SkipIndexSpec, TableEngineSpec, TypeFamilySpec } from "../clickhouse/catalog-types.js";`,
+        `import type { CodecName, ColumnTypeName, DatabaseEngineName, MergeTreeSettings, QuerySettings, SkipIndexType, TableEngineName } from "./clickhouse-types";`,
+        "",
+        `export * from "./clickhouse-types";`,
+        "",
+        `export const TABLE_ENGINES: Readonly<Record<TableEngineName, TableEngineSpec>> = {`,
+        ...engineLines,
+        "};",
+        "",
+        `export const DATABASE_ENGINES: Readonly<Record<DatabaseEngineName, DatabaseEngineSpec>> = {`,
+        ...catalog.databaseEngines.map((e) => `  ${key(e.name)}: ${JSON.stringify({ syntax: e.syntax, summary: e.summary })},`),
+        "};",
+        "",
+        `export const TYPE_FAMILIES: Readonly<Record<ColumnTypeName, TypeFamilySpec>> = {`,
+        ...catalog.typeFamilies.map((t) => `  ${key(t.name)}: ${JSON.stringify({ canonical: t.aliasOf ?? t.name, caseInsensitive: t.caseInsensitive })},`),
+        "};",
+        "",
+        `export const CODECS: Readonly<Record<CodecName, CodecSpec>> = {`,
+        ...catalog.codecs.map((c) => {
+            const o = CODEC_OVERLAY[c.name];
+            const spec = {
+                compression: c.compression,
+                generic: c.generic,
+                encryption: c.encryption,
+                timeseries: c.timeseries,
+                experimental: c.experimental,
+                role: o.role,
+                parameters: o.parameters,
+                summary: c.summary,
+            };
+            return `  ${key(c.name)}: ${JSON.stringify(spec)},`;
+        }),
+        "};",
+        "",
+        `/** Every function the server has (\`system.functions\`, origin System), aliases included. */`,
+        `export const FUNCTIONS: Readonly<Record<string, FunctionSpec>> = {`,
+        ...catalog.functions.map((f) => `  ${key(f.name)}: ${JSON.stringify({ aggregate: f.aggregate, caseInsensitive: f.caseInsensitive, ...(f.aliasOf ? { aliasOf: f.aliasOf } : {}) })},`),
+        "};",
+        "",
+        `/** The aggregate function combinators (\`system.aggregate_function_combinators\`): the suffixes of \`countIf\`, \`sumState\`. */`,
+        `export const AGGREGATE_COMBINATORS: readonly string[] = ${JSON.stringify(catalog.aggregateCombinators.map((c) => c.name))};`,
+        "",
+        `/** The table functions (\`system.table_functions\`): \`numbers\`, \`remote\`, \`s3\`. */`,
+        `export const TABLE_FUNCTIONS: readonly string[] = ${JSON.stringify(catalog.tableFunctions)};`,
+        "",
+        `export const SKIP_INDEX_TYPES: Readonly<Record<SkipIndexType, SkipIndexSpec>> = {`,
+        ...indexLines,
+        "};",
+        "",
+        settingsTable("MERGE_TREE_SETTINGS", "keyof MergeTreeSettings", catalog.mergeTreeSettings),
+        "",
+        settingsTable("QUERY_SETTINGS", "keyof QuerySettings", catalog.querySettings),
+        "",
+    ].join("\n");
+    return { declarations, tables, notes };
+}

package/dist/codegen/coverage-page.js

@@ -0,0 +1,70 @@
+/**
+ * `docs/pages/object-coverage.mdx`, written from the coverage tables
+ * (`../coverage/<dialect>.json`) by `npm run docs`, so the page cannot
+ * disagree with what import refuses.
+ */
+import { KIND_COVERAGE, PINNED_COMMANDS } from "../coverage.js";
+const cell = (s) => s.replace(/\|/g, "\\|").replace(/[{}<>]/g, (c) => `\\${c}`);
+const STATUS = { declared: "declared", refused: "refused", "out-of-scope": "out of scope" };
+function table(dialect) {
+    const rows = [...KIND_COVERAGE[dialect]].sort((a, b) => (a.command < b.command ? -1 : a.command > b.command ? 1 : 0));
+    return [
+        "| Command | Kind | Status | Declared by, or why not |",
+        "|---|---|---|---|",
+        ...rows.map((r) => {
+            const command = `\`${r.command}\`${r.form ? ` (${cell(r.form)})` : ""}`;
+            const why = r.status === "declared" ? `\`${r.entity}\`${r.note ? `: ${cell(r.note)}` : ""}` : cell(r.reason ?? "");
+            return `| ${command} | ${cell(r.kind)} | ${STATUS[r.status]} | ${why} |`;
+        }),
+    ].join("\n");
+}
+function counts(dialect) {
+    const rows = KIND_COVERAGE[dialect];
+    const n = (s) => rows.filter((r) => r.status === s).length;
+    return `${n("declared")} declared, ${n("refused")} refused, ${n("out-of-scope")} out of scope`;
+}
+export function renderCoveragePage() {
+    const pg = PINNED_COMMANDS.postgres.map((p) => p.version).join(", ");
+    const ch = PINNED_COMMANDS.clickhouse.map((p) => p.version).join(", ");
+    return `---
+title: "Object Coverage"
+label: "Object Coverage"
+description: "Every kind of object each pinned Postgres and ClickHouse server can create, marked declared, refused with a reason, or out of scope, and what import does with each"
+diataxis: reference
+order: 3
+---
+
+{/* Written by src/codegen/coverage-page.ts from src/coverage/postgres.json and src/coverage/clickhouse.json. Edit the tables, then run npm run docs. */}
+
+Each row is a \`CREATE\` command a pinned server has, or one form of a command where chant treats the forms apart (\`CREATE TYPE ... AS ENUM\` is declared, \`CREATE TYPE ... AS RANGE\` is not). The commands come from the servers' own sources at each pinned version (\`scripts/sql-create-commands.ts\`), and a test fails when a pinned server has a command with no row here, so a new server release cannot add a kind nobody looked at.
+
+- **declared**: a tag declares it, and import writes it as that declaration.
+- **refused**: no declaration holds it yet. \`chant import --from <env>\` fails when the server has one, naming every such object (kind, schema, name) at once, and \`chant import schema.sql\` fails on the statement. Nothing is left out without a word.
+- **out of scope**: the environment's, not a schema's: import never reads it.
+
+\`--skip-undeclarable\` (\`--parser-option skipUndeclarable\` for a file) imports everything else and lists the refused objects as warnings, for an adopter who keeps those objects by hand for now:
+
+\`\`\`bash
+chant import --from prod --skip-undeclarable
+chant import schema.sql --parser-option skipUndeclarable
+\`\`\`
+
+What a managed provider, an extension or the server's configuration files define is never refused: an extension's aggregates and casts are the extension's, declared with it, and a quota from \`users.xml\` is the server's. An import with \`--type\`, \`--name\` or \`--owned\` asks only for objects chant can hold, and does not look for the rest.
+
+## Postgres
+
+Pinned majors: ${pg}. ${counts("postgres")}.
+
+${table("postgres")}
+
+## ClickHouse
+
+Pinned release: ${ch}. ${counts("clickhouse")}.
+
+${table("clickhouse")}
+
+## Declaring a refused kind
+
+A kind becomes declarable in three steps: a tag that declares it, the catalog reader reading it (\`src/postgres/live/catalog.ts\`, \`src/clickhouse/live/catalog.ts\`), and its row flipped to \`declared\` with the entity type, its reason taken out. The row's flip is what stops import refusing the kind. After moving a pin, run \`npx tsx scripts/sql-create-commands.ts\` and give each new command a row.
+`;
+}

package/dist/codegen/docs-cli.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env tsx
+import { generateDocs } from "./docs.js";
+await generateDocs({ verbose: true });

package/dist/codegen/docs.js

@@ -0,0 +1,222 @@
+/**
+ * Documentation generation for the sql lexicon: the generated reference pages
+ * from the core docs pipeline, plus the authored pages in docs/pages/.
+ */
+import { dirname, join } from "path";
+import { fileURLToPath } from "url";
+import { writeFileSync } from "fs";
+import { docsPipeline, writeDocsSite } from "@intentius/chant/codegen/docs";
+import { renderClassifierPage } from "./classifier-page.js";
+import { renderPostgresClassifierPage } from "./postgres-classifier-page.js";
+import { renderCoveragePage } from "./coverage-page.js";
+const overview = `The sql lexicon declares database schema in TypeScript. It has two dialects,
+each a subpath of the one package: ClickHouse at
+\`@intentius/chant-lexicon-sql/clickhouse\` and Postgres at
+\`@intentius/chant-lexicon-sql/postgres\`. A declaration is the database's own
+\`CREATE\` statement in a tagged template, and an interpolated object or
+column stays a reference. Install it with
+\`npm install --save-dev @intentius/chant @intentius/chant-lexicon-sql\`; each
+dialect's Getting Started page runs the example the lexicon ships.
+
+\`\`\`ts
+import { database, table, view } from "@intentius/chant-lexicon-sql/clickhouse";
+
+export const analytics = database\`CREATE DATABASE analytics ENGINE = Atomic\`;
+
+export const events = table\`
+  CREATE TABLE \${analytics}.events (
+    user_id  UUID,
+    kind     LowCardinality(String),
+    ts       DateTime
+  )
+  ENGINE = MergeTree
+  ORDER BY (user_id, kind, ts)\`;
+\`\`\`
+
+\`\`\`ts
+import { schema, table, index } from "@intentius/chant-lexicon-sql/postgres";
+
+export const app = schema\`CREATE SCHEMA app\`;
+
+export const users = table\`
+  CREATE TABLE \${app}.users (
+    id    bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    email text NOT NULL UNIQUE
+  )\`;
+
+export const usersEmail = index\`
+  CREATE INDEX CONCURRENTLY users_email_lower_idx ON \${users} (lower(\${users.columns.email}))\`;
+\`\`\`
+
+## Spec-true per dialect
+
+Each dialect is true to its own database. For SQL the spec is the database's
+own DDL, so a ClickHouse declaration is a ClickHouse \`CREATE\` statement and a
+Postgres declaration a Postgres one. The names a declaration can use (engines,
+codecs and settings for ClickHouse; types, access methods, storage parameters
+and extensions for Postgres) are generated from a pinned server's own catalog:
+ClickHouse's \`system.*\` tables, and Postgres's \`pg_catalog\` at one pinned
+release of each supported major. There is no neutral schema model that the
+dialects are translated from.
+
+The reason is that the databases differ exactly where a schema tool does its
+work. ClickHouse has no foreign keys and no transactional DDL, and it cannot
+change a table's sort key, partition key or engine in place: the rows have to
+be copied into a new table. Postgres makes most changes in place and inside a
+transaction, and what a change costs there is the lock it takes: adding a
+column with a constant default is a catalog change, a type change that
+rewrites the table blocks every reader until it finishes, and an index built
+without \`CONCURRENTLY\` blocks every writer. A neutral model would have to hide
+those differences, so a plan could not say what a change will cost, or carry
+every dialect's features and check each one against the target anyway.
+
+So each dialect has its own types, parser, lint rules and post-synth checks,
+change classifier, applier, migration Op, composites and skills. What the two
+share is the way of working:
+
+| | ClickHouse | Postgres |
+|---|---|---|
+| Declaring | \`database\`, \`table\`, \`view\` | \`schema\`, \`table\`, \`index\`, \`view\`, \`sequence\`, \`type\`, \`domain\`, \`extension\` |
+| Types from | one pinned \`clickhouse-server\` release | one pinned \`postgres\` release per major, 14 to 18 |
+| Rule ids | SQLCH001-003, SQLCH101-120, SQLCH200-250 | SQLPG001-004, SQLPG101-118, SQLPG200-270 |
+| Change classes | metadata only, background rewrite, rebuild | metadata only, validates under a weaker lock, needs \`CONCURRENTLY\`, \`ACCESS EXCLUSIVE\` rewrite or scan, expand and contract |
+| Applier (\`ApplyOp\` target) | \`clickhouseApply\` (\`clickhouse\`) | \`postgresApply\` (\`postgres\`) |
+| What a plan refuses runs as | \`ClickHouseRebuildOp\` | \`PostgresMigrationOp\` |
+| Ownership marker | a trailer on the object's comment | a trailer on the object's comment |
+
+References, dependency order and column lineage work the same way in both, as
+do \`chant sql diff\` and \`chant sql plan\` (they read the build's
+\`dialect\`), import with \`chant import --from <env>\`, the \`-- previously:\`
+rename hint, and the comment trailer that marks what chant owns.
+
+One build holds one dialect. A project that declares both ClickHouse and
+Postgres objects fails the build naming one of each; a Postgres database and
+a ClickHouse database are two projects, which a workspace can hold as two
+members. \`sql.dialect\` in \`chant.config.ts\` says which dialect a project is
+when nothing declared says so, as for an import into an empty project.
+
+## Pages
+
+ClickHouse:
+
+- [Getting Started with ClickHouse](./getting-started/): declare a schema,
+  build it, apply it to a local server and plan a change against it.
+- [Declaring ClickHouse Tables and Views](./clickhouse-ddl/): databases,
+  tables, views and materialized views, and what each kind of interpolation
+  means.
+- [Importing a ClickHouse Server](./importing/): \`chant import --from <env>\`
+  from \`SHOW CREATE\`.
+- [Planning and the Change Classifier](./change-classifier/): every change
+  classified as metadata only, a background rewrite or a rebuild.
+- [Applying to a ClickHouse Server](./applying/): \`clickhouseApply\`, the
+  ownership marker, prune and the local server.
+- [Rebuilding a Table](./rebuild/): \`ClickHouseRebuildOp\` for the changes
+  ClickHouse cannot make in place.
+- [Where the ClickHouse Types Come From](./clickhouse-catalog/): the pinned
+  server and how to move the pin.
+
+Postgres:
+
+- [Getting Started with Postgres](./postgres-getting-started/): declare a
+  schema, apply it to a local server, add a column and an index, and see a
+  rename refused.
+- [Declaring Postgres Objects](./postgres-ddl/): the eight tags,
+  constraints, \`nextval(\${seq})\`, \`COMMENT ON\` and \`literal()\`.
+- [Importing a Postgres Server](./postgres-importing/): canonical text from
+  the catalog's printers, and the objects that belong to someone else.
+- [Postgres Locks and the Change Classifier](./postgres-change-classifier/):
+  every change classified by the lock it takes.
+- [Applying to a Postgres Server](./postgres-applying/): transactions,
+  \`CONCURRENTLY\`, timeouts, ownership, prune and the local server.
+- [Migrating a Column](./postgres-migration/): \`PostgresMigrationOp\`, the
+  expand-and-contract Op for a column rename or type change.
+- [Managed Providers](./postgres-providers/): \`sql.provider\`, SQLPG004, and
+  what each provider's data has not verified.
+- [The Postgres Pin and Supported Majors](./postgres-majors/): one pin per
+  major, \`sql.postgresMajor\`, and moving a pin.
+
+Both dialects:
+
+- [References and Lineage](./references-and-lineage/): how references become
+  dependency order and column-level lineage.
+- [Composites](./composites/): five ClickHouse and five Postgres composites.
+- [Lint Rules and Checks](./lint-rules/): every lint rule and post-synth
+  check.
+- [Object Coverage](./object-coverage/): every kind of object the pinned
+  servers can create, declared, refused or out of scope, and what import
+  does with each.
+
+## Schema changes and \`chant migrate\`
+
+\`chant migrate\` translates a file from one lexicon's format into another's,
+such as a GitHub Actions workflow into GitLab CI. It does not run schema
+migrations, and the sql lexicon registers nothing for it. A schema change is a
+change to the declarations: \`chant sql diff\` or \`chant sql plan\` classifies
+it, the dialect's applier makes the changes it can make in place, and a change
+it cannot make runs as a gated migration Op: \`ClickHouseRebuildOp\` (see
+[Rebuilding a Table](./rebuild/)) or \`PostgresMigrationOp\` (see
+[Migrating a Column](./postgres-migration/)).
+`;
+const outputFormat = `A build writes two files, beside each other. Which pair depends on the
+dialect the project declares.
+
+- The primary output (the \`-o\` file, \`dist/schema.json\` by convention), one
+  JSON document. \`dialect\` is \`clickhouse\` or \`postgres\`. \`applyOrder\`
+  lists the export names in the order the objects have to be created.
+  \`objects\` holds one entry per declared object, in that order: its export
+  name (\`export\`), its type, its SQL name (\`sqlName\`), its parsed
+  definition, \`dependsOn\`, and its \`ddl\`. References are written as export
+  names and lineage as \`export.column\`.
+  - ClickHouse: the types are \`ClickHouse::Database\`, \`::Table\`, \`::View\`
+    and \`::MaterializedView\`, and the definition holds columns, engine, keys,
+    TTL, settings, indexes, projections and constraints, and for a view its
+    \`select\`, \`reads\`, \`to\` and \`lineage\`.
+  - Postgres: the document also carries \`postgresMajor\`, the major the
+    project targets (\`sql.postgresMajor\`, else 18, the newest pinned). The
+    types are \`Postgres::Schema\`, \`::Table\`, \`::Index\`, \`::View\`,
+    \`::MaterializedView\`, \`::Sequence\`, \`::Enum\`, \`::Domain\` and
+    \`::Extension\`. A table's definition holds its columns, primary key,
+    uniques, checks, foreign keys, exclusions and the rest of its clauses; a
+    view's its \`query\`, \`reads\` and \`lineage\`; an index's its table,
+    method and elements. \`dependsOn\` names the objects and the columns the
+    statement references.
+- \`clickhouse.sql\` or \`postgres.sql\`: every statement in \`applyOrder\`, each
+  ending in \`;\`, written byte for byte as declared, with a Postgres
+  template's \`COMMENT ON\` statements after its \`CREATE\`.
+
+The order follows the references: a table after its database or schema and the
+types and sequences it uses, a view after the tables it reads, an index after
+its table, a materialized view after its \`TO\` target. Ties go by export name,
+so the output does not depend on the order files were found in. A reference
+cycle fails the build and names the cycle.
+
+\`chant sql diff\`, \`chant sql plan\`, the post-synth checks and the appliers
+read the JSON document. Neither file carries chant's ownership marker: the
+applier adds it to each object's comment when it creates the object (see
+[Applying to a ClickHouse Server](../applying/) and
+[Applying to a Postgres Server](../postgres-applying/)).
+`;
+export async function generateDocs(opts) {
+    const pkgDir = dirname(dirname(dirname(fileURLToPath(import.meta.url))));
+    writeFileSync(join(pkgDir, "docs", "pages", "change-classifier.mdx"), renderClassifierPage());
+    writeFileSync(join(pkgDir, "docs", "pages", "postgres-change-classifier.mdx"), renderPostgresClassifierPage());
+    writeFileSync(join(pkgDir, "docs", "pages", "object-coverage.mdx"), renderCoveragePage());
+    const config = {
+        name: "sql",
+        displayName: "SQL",
+        description: "Database schema as typed declarations in each database's own DDL: ClickHouse and Postgres",
+        distDir: join(pkgDir, "dist"),
+        outDir: join(pkgDir, "docs"),
+        basePath: process.env.DOCS_BASE_PATH ?? "/chant/lexicons/sql/",
+        overview,
+        outputFormat,
+        serviceFromType: (type) => type.split("::")[0] ?? "SQL",
+        srcDir: join(pkgDir, "src"),
+        examplesDir: join(pkgDir, "examples"),
+    };
+    const result = docsPipeline(config);
+    writeDocsSite(config, result);
+    if (opts?.verbose) {
+        console.error(`Generated ${result.pages.size} documentation pages`);
+    }
+}

package/dist/codegen/generate-cli.js

@@ -0,0 +1,13 @@
+#!/usr/bin/env tsx
+import { generate, generatePostgres, writeGeneratedFiles } from "./generate.js";
+// `npm run generate -- --force` reads the pinned servers even when the snapshots match the pins;
+// `--major=18` limits the Postgres read to one major.
+const force = process.argv.includes("--force");
+const majors = process.argv
+    .filter((a) => a.startsWith("--major="))
+    .map((a) => Number.parseInt(a.slice("--major=".length), 10));
+const result = await generate({ verbose: true, force });
+const postgres = await generatePostgres({ force, ...(majors.length > 0 ? { majors } : {}) });
+for (const note of postgres.notes)
+    console.error(`[sql] ${note}`);
+writeGeneratedFiles(result, undefined, postgres);

package/dist/codegen/generate.js

@@ -0,0 +1,97 @@
+/**
+ * The sql lexicon's generation pipeline.
+ *
+ * Two dialects. ClickHouse's "schema" is the catalog a pinned
+ * `clickhouse-server` reports from `system.*`, committed as
+ * `src/spec/clickhouse-catalog.snapshot.json` (chant #3195). `fetchSchemas`
+ * reads that snapshot, and starts the pinned server only on a pin move; see
+ * `src/spec/fetch.ts`.
+ *
+ * Outputs, under `src/generated/`:
+ *
+ * - `clickhouse-types.ts` (and `index.d.ts`, the packaged copy): unions of every
+ *   engine, type family, codec, skip index type, format and function name, and
+ *   the `MergeTreeSettings` / `QuerySettings` interfaces.
+ * - `clickhouse.ts`: the tables lint and the LSP read, with the hand-written
+ *   overlays (engine argument kinds, codec parameters) merged in.
+ * - `lexicon-sql.json`: the entity registry.
+ */
+import { generatePipeline, writeGeneratedArtifacts, } from "@intentius/chant/codegen/generate";
+import { NamingStrategy } from "@intentius/chant/codegen/naming";
+import { dirname } from "path";
+import { fileURLToPath } from "url";
+import { CATALOG_KEY, fetchSchemas } from "../spec/fetch.js";
+import { parseCatalog } from "../spec/catalog.js";
+import { renderClickHouseModule } from "./clickhouse-module.js";
+import { buildRegistry } from "./registry.js";
+import { fetchCatalogs } from "../spec/postgres-fetch.js";
+import { renderPostgresModule } from "./postgres-module.js";
+export async function generate(opts = {}) {
+    let notes = [];
+    let tables = "";
+    const result = await generatePipeline({
+        fetchSchemas: (fetchOpts) => fetchSchemas({ force: fetchOpts.force }),
+        parseSchema: (typeName, data) => {
+            if (typeName !== CATALOG_KEY)
+                return null;
+            const catalog = parseCatalog(data.toString("utf-8"));
+            return {
+                typeName,
+                catalog,
+                propertyTypes: [{ name: "MergeTreeSettings" }, { name: "QuerySettings" }],
+                enums: [],
+            };
+        },
+        createNaming: (results) => new NamingStrategy(results.map((r) => ({ typeName: r.typeName, propertyTypes: r.propertyTypes })), {
+            priorityNames: {},
+            priorityAliases: {},
+            priorityPropertyAliases: {},
+            serviceAbbreviations: {},
+            shortName: (t) => t.split("::").pop(),
+            serviceName: (t) => t.split("::")[0] ?? t,
+        }),
+        generateRegistry: () => buildRegistry(),
+        generateTypes: (results) => {
+            const catalog = results.find((r) => r.typeName === CATALOG_KEY)?.catalog;
+            if (!catalog)
+                throw new Error("no ClickHouse catalog was parsed");
+            const rendered = renderClickHouseModule(catalog);
+            notes = rendered.notes;
+            tables = rendered.tables;
+            return rendered.declarations;
+        },
+        generateRuntimeIndex: () => tables,
+    }, opts);
+    if (result.warnings.length > 0) {
+        throw new Error(`sql generation failed:\n  ${result.warnings.map((w) => `${w.file}: ${w.error}`).join("\n  ")}`);
+    }
+    if (opts.verbose)
+        for (const note of notes)
+            console.error(`[sql] ${note}`);
+    return result;
+}
+/**
+ * Generate the Postgres types: the union of the catalogs of every supported
+ * major (`src/spec/postgres-catalog-<major>.snapshot.json`), rendered with the
+ * overlays. A server is started only for a major whose pin moved, or with
+ * `force`; see `src/spec/postgres-fetch.ts`. The types stay internal to the
+ * package until the `/postgres` subpath has entities to export with them.
+ */
+export async function generatePostgres(opts = {}) {
+    const catalogs = await fetchCatalogs(opts);
+    return renderPostgresModule(catalogs);
+}
+/** Write the generated files under `src/generated/`. */
+export function writeGeneratedFiles(result, pkgDir, postgres) {
+    const baseDir = pkgDir ?? dirname(dirname(dirname(fileURLToPath(import.meta.url))));
+    writeGeneratedArtifacts({
+        baseDir,
+        files: {
+            "lexicon-sql.json": result.lexiconJSON,
+            "index.d.ts": result.typesDTS,
+            "clickhouse-types.ts": result.typesDTS,
+            "clickhouse.ts": result.indexTS,
+            ...(postgres ? { "postgres-types.ts": postgres.declarations, "postgres.ts": postgres.tables } : {}),
+        },
+    });
+}