@intentius/chant-lexicon-sql 0.0.0-stage → 0.112.1

package/src/clickhouse/live/catalog.ts

@@ -0,0 +1,96 @@
+/**
+ * Reading a live server's schema: the databases and the tables, views and
+ * materialized views in them, from `system.databases`, `system.tables` and
+ * `SHOW CREATE`.
+ */
+
+import { clickhouseQuery } from "../http";
+import type { ClickHouseTarget } from "./bind";
+import { CLICKHOUSE_ENTITY_TYPES, type ClickHouseEntityType } from "../entities";
+import { isChantWorkingObject, stripMarkerFromStatement } from "../ownership";
+
+/** The server's own databases, never part of a declared schema. */
+export const SYSTEM_DATABASES = new Set(["system", "information_schema", "INFORMATION_SCHEMA"]);
+
+export interface LiveObject {
+  type: ClickHouseEntityType;
+  database?: string;
+  name: string;
+  /** The engine `system.tables` / `system.databases` report: `MergeTree`, `View`, `MaterializedView`, `Atomic`. */
+  engine: string;
+  uuid?: string;
+  /** The object's comment as the server holds it, chant's ownership trailer included (`../ownership.ts`). */
+  comment?: string;
+  /** `SHOW CREATE` as the server prints it, less chant's ownership trailer, so it reads as the declaration. */
+  statement: string;
+}
+
+const quote = (s: string) => `'${s.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
+const ident = (s: string) => `\`${s.replace(/`/g, "``")}\``;
+
+/** The entity type a `system.tables` engine stands for. */
+export function entityTypeOfEngine(engine: string): ClickHouseEntityType {
+  if (engine === "View") return CLICKHOUSE_ENTITY_TYPES.view;
+  if (engine === "MaterializedView") return CLICKHOUSE_ENTITY_TYPES.materializedView;
+  return CLICKHOUSE_ENTITY_TYPES.table;
+}
+
+/**
+ * Every object in the target's databases, sorted by database then name.
+ * A materialized view's inner table (`.inner_id.<uuid>`, `.inner.<name>`)
+ * belongs to its view and is left out, as are temporary tables and
+ * dictionaries (not declared in slice 1).
+ *
+ * So are chant's own working objects (`../ownership.ts`
+ * `isChantWorkingObject`): a rebuild migration's new, dual-write and retained
+ * tables, and the effect receipts database. None of them is a declaration, so
+ * a plan, an import or a prune that saw them would propose dropping, importing
+ * or pruning the very objects a rebuild in progress depends on.
+ */
+export async function readLiveSchema(target: ClickHouseTarget, opts: { withStatements?: boolean } = {}): Promise<LiveObject[]> {
+  const q = <T>(sql: string) => clickhouseQuery<T>(target.endpoint, sql);
+  const scope = target.databases
+    ? `IN (${target.databases.map(quote).join(", ")})`
+    : `NOT IN (${[...SYSTEM_DATABASES].map(quote).join(", ")})`;
+
+  const databases = await q<{ name: string; engine: string; uuid: string; comment: string }>(
+    `SELECT name, engine, toString(uuid) AS uuid, comment FROM system.databases WHERE name ${scope} ORDER BY name`,
+  );
+  const tables = await q<{ database: string; name: string; engine: string; uuid: string; comment: string }>(
+    `SELECT database, name, engine, toString(uuid) AS uuid, comment FROM system.tables ` +
+      `WHERE database ${scope} AND NOT is_temporary AND name NOT LIKE '.inner%' AND engine != 'Dictionary' ` +
+      `ORDER BY database, name`,
+  );
+
+  const working = new Set(databases.filter((d) => isChantWorkingObject(d.comment)).map((d) => d.name));
+  const out: LiveObject[] = [];
+  const statement = async (kind: "DATABASE" | "TABLE", what: string) =>
+    opts.withStatements === false
+      ? ""
+      : stripMarkerFromStatement((await q<{ statement: string }>(`SHOW CREATE ${kind} ${what}`))[0]?.statement ?? "");
+
+  for (const d of databases) {
+    if (working.has(d.name)) continue;
+    out.push({
+      type: CLICKHOUSE_ENTITY_TYPES.database,
+      name: d.name,
+      engine: d.engine,
+      ...(d.uuid && !/^0{8}-/.test(d.uuid) ? { uuid: d.uuid } : {}),
+      ...(d.comment ? { comment: d.comment } : {}),
+      statement: await statement("DATABASE", ident(d.name)),
+    });
+  }
+  for (const t of tables) {
+    if (working.has(t.database) || isChantWorkingObject(t.comment)) continue;
+    out.push({
+      type: entityTypeOfEngine(t.engine),
+      database: t.database,
+      name: t.name,
+      engine: t.engine,
+      ...(t.uuid && !/^0{8}-/.test(t.uuid) ? { uuid: t.uuid } : {}),
+      ...(t.comment ? { comment: t.comment } : {}),
+      statement: await statement("TABLE", `${ident(t.database)}.${ident(t.name)}`),
+    });
+  }
+  return out;
+}

package/src/clickhouse/live/describe-resources.ts

@@ -0,0 +1,98 @@
+/**
+ * `describeResources()` for the ClickHouse dialect: which declared databases,
+ * tables and views exist on the environment's server.
+ *
+ * One read of `system.databases` and `system.tables` per run, then a lookup
+ * per declared entity, so N entities are two queries, not N. An object the
+ * catalog does not list is absent, and the `queried` address says which
+ * server and name were asked. A server that cannot be reached, or refuses the
+ * credentials, leaves every entity unobserved with that reason: never absent.
+ *
+ * Ownership is read from chant's marker, the trailer the applier appends to
+ * the object's comment (`../ownership.ts`): `owned` when the comment carries
+ * it, `foreign` when it does not. With `owned: true` a foreign object is
+ * withheld as `filtered`, which says it exists and is not chant's, never that
+ * it is absent.
+ */
+
+import {
+  observeEntities,
+  type DeclaredEntity,
+  type DescribeResourcesResult,
+  type EntityObservation,
+  type ObserverAdapter,
+} from "@intentius/chant/observation";
+import { bindClickHouse, classifyClickHouseFailure, type BindOptions, type ClickHouseTarget } from "./bind";
+import { readLiveSchema, type LiveObject } from "./catalog";
+import { CLICKHOUSE_ENTITY_TYPES } from "../entities";
+import { isChantManaged, readMarker, stripMarker } from "../ownership";
+
+interface Bound {
+  target: ClickHouseTarget;
+  byKey: Map<string, LiveObject>;
+}
+
+const keyOf = (database: string | undefined, name: string) => `${database ?? ""}.${name}`;
+
+/** The database and name a declared entity is created as. */
+export function declaredAddress(entity: DeclaredEntity, defaultDatabase: string): { database?: string; name: string } {
+  const name = String(entity.props.name ?? "");
+  if (entity.type === CLICKHOUSE_ENTITY_TYPES.database) return { name };
+  return { database: typeof entity.props.database === "string" ? entity.props.database : defaultDatabase, name };
+}
+
+function adapter(options: BindOptions & { owned?: boolean }): ObserverAdapter<Bound> {
+  return {
+    async bind() {
+      const target = await bindClickHouse(options);
+      const byKey = new Map<string, LiveObject>();
+      for (const o of await readLiveSchema(target, { withStatements: false })) {
+        byKey.set(o.type === CLICKHOUSE_ENTITY_TYPES.database ? keyOf(undefined, o.name) : keyOf(o.database, o.name), o);
+      }
+      return { target, byKey };
+    },
+    classifyBindFailure: (err) => classifyClickHouseFailure(err),
+    async read({ target, byKey }, entity): Promise<EntityObservation> {
+      if (!entity.type.startsWith("ClickHouse::")) {
+        return { unobserved: { reason: "unsupported-kind", detail: entity.type } };
+      }
+      const { database, name } = declaredAddress(entity, target.defaultDatabase);
+      const queried = `${target.endpoint.url} ${database ? `${database}.` : ""}${name}`;
+      const live = byKey.get(keyOf(database, name));
+      if (!live) return { absent: true, queried };
+      const owned = isChantManaged(live.comment);
+      if (options.owned && !owned) {
+        return { unobserved: { reason: "filtered", detail: "the object's comment carries no chant ownership marker and owned was requested" }, queried };
+      }
+      const marker = readMarker(live.comment);
+      const comment = live.comment ? stripMarker(live.comment) : "";
+      return {
+        present: {
+          type: live.type,
+          physicalId: live.uuid ?? (database ? `${database}.${name}` : name),
+          status: live.engine,
+          attributes: { engine: live.engine, ...(live.uuid ? { uuid: live.uuid } : {}), ...(comment ? { comment } : {}) },
+          ownership: owned ? "owned" : "foreign",
+          ...(owned && marker?.stack ? { marker: { stack: marker.stack, ...(marker.env ? { env: marker.env } : {}) } } : {}),
+        },
+        queried,
+      };
+    },
+  };
+}
+
+export async function describeResources(
+  options: {
+    environment: string;
+    entityNames: string[];
+    entities: Map<string, { entityType: string; props: Record<string, unknown> }>;
+    /** Withhold objects that do not carry chant's ownership marker. */
+    owned?: boolean;
+  } & Omit<BindOptions, "environment">,
+): Promise<DescribeResourcesResult> {
+  const declared: DeclaredEntity[] = options.entityNames.map((name) => {
+    const entity = options.entities.get(name);
+    return { name, type: entity?.entityType ?? "", props: entity?.props ?? {} };
+  });
+  return observeEntities(declared, adapter(options));
+}

package/src/clickhouse/live/live.test.ts

@@ -0,0 +1,143 @@
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { describeObservationConformance } from "@intentius/chant-test-utils";
+import { normalizeObservation } from "@intentius/chant/observation";
+import { resolveClickHouseTarget, isUnresolvedTarget } from "./bind";
+import { describeResources } from "./describe-resources";
+import { fakeClickHouse, type FakeServer } from "../testing/fake-server";
+import { database, table, view } from "../entities";
+import { SQL_OWNERSHIP_CHANNEL } from "../ownership";
+
+const analytics = database`CREATE DATABASE analytics ENGINE = Atomic`;
+const events = table`CREATE TABLE ${analytics}.events (id UInt64) ENGINE = MergeTree ORDER BY id`;
+const missing = table`CREATE TABLE ${analytics}.missing (id UInt64) ENGINE = Log`;
+const inDefault = table`CREATE TABLE plain (id UInt64) ENGINE = Log`;
+const byId = view`CREATE VIEW analytics.by_id AS SELECT id FROM ${events}`;
+
+const LIVE = [
+  { name: "analytics", engine: "Atomic", statement: "CREATE DATABASE analytics\nENGINE = Atomic" },
+  {
+    database: "analytics",
+    name: "events",
+    engine: "MergeTree",
+    comment: "Raw events [chant managed-by=chant stack=shop env=prod]",
+    statement: "CREATE TABLE analytics.events\n(\n    `id` UInt64\n)\nENGINE = MergeTree\nORDER BY id\nSETTINGS index_granularity = 8192\nCOMMENT 'Raw events [chant managed-by=chant stack=shop env=prod]'",
+  },
+  { database: "analytics", name: "by_id", engine: "View", statement: "CREATE VIEW analytics.by_id\n(\n    `id` UInt64\n)\nAS SELECT id\nFROM analytics.events" },
+  { database: "default", name: "plain", engine: "Log", statement: "CREATE TABLE default.plain\n(\n    `id` UInt64\n)\nENGINE = Log" },
+];
+
+const entities = new Map(
+  Object.entries({ analytics, events, missing, inDefault, byId }).map(([k, v]) => [k, { entityType: v.entityType, props: v.props as unknown as Record<string, unknown> }]),
+);
+const entityNames = [...entities.keys()];
+
+let ok: FakeServer;
+let refused: FakeServer;
+beforeAll(async () => {
+  ok = await fakeClickHouse(LIVE);
+  refused = await fakeClickHouse([], { fail: { status: 516, body: "Code: 516. DB::Exception: default: Authentication failed. (AUTHENTICATION_FAILED)" } });
+});
+afterAll(async () => {
+  await ok.close();
+  await refused.close();
+});
+
+const run = (url: string | undefined, owned?: boolean) =>
+  describeResources({ environment: "test", entityNames, entities, config: {}, env: url ? { CLICKHOUSE_URL: url } : {}, ...(owned ? { owned } : {}) });
+
+describe("binding an environment to a server", () => {
+  test("a profile is the server, its credentials read from the variables it names", () => {
+    const t = resolveClickHouseTarget({
+      environment: "prod",
+      config: { sql: { profiles: { prod: { url: "https://ch:8443/", user: { env: "U" }, password: { env: "P" }, databases: ["analytics"] } } } },
+      env: { U: "reader", P: "pw" },
+    });
+    expect(t).toEqual({
+      endpoint: { url: "https://ch:8443", user: "reader", password: "pw" },
+      source: "sql.profiles.prod",
+      databases: ["analytics"],
+      defaultDatabase: "default",
+    });
+  });
+
+  test("a credential variable the profile names but nobody set is no-credentials", () => {
+    const t = resolveClickHouseTarget({ environment: "prod", config: { sql: { profiles: { prod: { url: "x", password: { env: "P" } } } } }, env: {} });
+    expect(isUnresolvedTarget(t) && t.reason).toBe("no-credentials");
+  });
+
+  test("no profile and no CLICKHOUSE_URL is no-binding", () => {
+    const t = resolveClickHouseTarget({ environment: "dev", config: {}, env: {} });
+    expect(isUnresolvedTarget(t) && t.reason).toBe("no-binding");
+  });
+});
+
+describe("describeResources", () => {
+  test("reports present objects with their engine, an undeclared-on-server one absent, and where it asked", async () => {
+    const r = normalizeObservation(await run(ok.url));
+    expect(Object.keys(r.resources).sort()).toEqual(["analytics", "byId", "events", "inDefault"]);
+    expect(r.resources.events).toMatchObject({ type: "ClickHouse::Table", status: "MergeTree", ownership: "owned" });
+    expect(r.resources.byId).toMatchObject({ type: "ClickHouse::View", status: "View", ownership: "foreign" });
+    expect(r.unobserved).toEqual({});
+    expect(r.queried?.missing).toBe(`${ok.url} analytics.missing`);
+    expect(r.queried?.inDefault).toBe(`${ok.url} default.plain`);
+  });
+
+  test("an owned object carries the marker's stack and env, and its comment without the trailer (#3208)", async () => {
+    const r = normalizeObservation(await run(ok.url));
+    expect(r.resources.events!.marker).toEqual({ stack: "shop", env: "prod" });
+    expect(r.resources.events!.attributes).toMatchObject({ comment: "Raw events" });
+    expect(r.resources.byId!.marker).toBeUndefined();
+  });
+
+  test("owned withholds a foreign object as filtered, never as absent (#3208)", async () => {
+    const r = normalizeObservation(await run(ok.url, true));
+    expect(Object.keys(r.resources)).toEqual(["events"]);
+    expect(Object.fromEntries(Object.entries(r.unobserved).map(([k, v]) => [k, v.reason]))).toEqual({
+      analytics: "filtered",
+      inDefault: "filtered",
+      byId: "filtered",
+    });
+  });
+
+  test("reads the catalog in two queries, whatever the number of entities", async () => {
+    const before = ok.queries.length;
+    await run(ok.url);
+    expect(ok.queries.length - before).toBe(2);
+  });
+});
+
+describeObservationConformance({
+  lexicon: "sql",
+  ownershipChannel: SQL_OWNERSHIP_CHANNEL,
+  scenarios: [
+    {
+      name: "owned: only what carries chant's marker",
+      declared: entityNames,
+      owned: true,
+      run: () => run(ok.url, true),
+      expectPresent: ["events"],
+      expectAbsent: ["missing"],
+      expectUnobserved: ["analytics", "inDefault", "byId"],
+      expectMarker: { events: { stack: "shop", env: "prod" } },
+    },
+    {
+      name: "a server holding some of the declared schema",
+      declared: entityNames,
+      run: () => run(ok.url),
+      expectPresent: ["analytics", "events", "inDefault", "byId"],
+      expectAbsent: ["missing"],
+    },
+    {
+      name: "a server that refuses the credentials",
+      declared: entityNames,
+      run: () => run(refused.url),
+      expectUnobserved: entityNames,
+    },
+    {
+      name: "no server bound",
+      declared: entityNames,
+      run: () => run(undefined),
+      expectUnobserved: entityNames,
+    },
+  ],
+});

package/src/clickhouse/live/roundtrip.e2e.test.ts

@@ -0,0 +1,93 @@
+/**
+ * The live round trip (#3197): a schema on one server is imported as chant
+ * declarations, built, and applied to a second, empty server; both servers'
+ * catalogs then print the same `SHOW CREATE` for every object, and exporting
+ * the second gives back the same declarations.
+ *
+ * Needs Docker; skips cleanly without it. Two throwaway
+ * `clickhouse/clickhouse-server` containers at the pin, removed afterwards
+ * even on failure.
+ */
+
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
+import { build } from "@intentius/chant/build";
+import type { SerializerResult } from "@intentius/chant/serializer";
+import { dockerAvailable, startScratchServer, type ScratchServer } from "../container";
+import { clickhouseQuery } from "../http";
+import { clickhouseImage } from "../../spec/pin";
+import { exportResources } from "../import/live-export";
+import { ClickHouseGenerator } from "../import/generator";
+import { splitStatements } from "../import/ir";
+import { readLiveSchema } from "./catalog";
+import { describeResources } from "./describe-resources";
+import { SCHEMA } from "../testing/roundtrip-schema";
+import { sqlPlugin } from "../../plugin";
+import { CLICKHOUSE_DDL_FILE, sqlSerializer } from "../../serializer";
+
+const enabled = await dockerAvailable();
+const projectDir = join(import.meta.dirname, "..", "..", "..", `.roundtrip-tmp-${process.pid}`);
+let source: ScratchServer | undefined;
+let target: ScratchServer | undefined;
+
+beforeAll(async () => {
+  if (!enabled) return;
+  [source, target] = await Promise.all([
+    startScratchServer(clickhouseImage(), { namePrefix: "chant-sql-rt-source" }),
+    startScratchServer(clickhouseImage(), { namePrefix: "chant-sql-rt-target" }),
+  ]);
+  for (const stmt of SCHEMA) await clickhouseQuery(source.endpoint, stmt);
+}, 600_000);
+
+afterAll(async () => {
+  await Promise.all([source?.stop(), target?.stop()]);
+  rmSync(projectDir, { recursive: true, force: true });
+});
+
+/** Every object's SHOW CREATE, keyed by `db.name`, with the UUIDs servers mint left out. */
+async function catalog(server: ScratchServer): Promise<Record<string, string>> {
+  const objects = await readLiveSchema({ endpoint: server.endpoint, source: "test", defaultDatabase: "default" });
+  return Object.fromEntries(
+    objects.map((o) => [`${o.database ?? ""}.${o.name}`, o.statement.replace(/ UUID '[0-9a-f-]+'/g, "")]),
+  );
+}
+
+describe.skipIf(!enabled)("import, build, apply, and the catalog is unchanged", () => {
+  test("the round trip", async () => {
+    const env = (s: ScratchServer) => ({ CLICKHOUSE_URL: s.endpoint.url });
+
+    // Import: the source server's schema as declarations.
+    const exported = await exportResources({ environment: "rt", config: {}, env: env(source!) });
+    const [file] = new ClickHouseGenerator().generate(exported);
+    mkdirSync(join(projectDir, "src"), { recursive: true });
+    writeFileSync(join(projectDir, "chant.config.ts"), 'export default { lexicons: ["sql"] };\n');
+    writeFileSync(join(projectDir, "src", file!.path), file!.content);
+
+    // Build: the declarations fold to the statements.
+    const result = await build(join(projectDir, "src"), [sqlSerializer], undefined, {
+      fold: true,
+      intrinsics: sqlPlugin.intrinsics!(),
+      lexicons: ["sql"],
+    });
+    expect(result.errors).toEqual([]);
+    const out = result.outputs.get("sql") as SerializerResult;
+
+    // Apply: run the statements, in the order the build wrote them, on the empty server.
+    for (const stmt of splitStatements(out.files![CLICKHOUSE_DDL_FILE]!)) await clickhouseQuery(target!.endpoint, stmt);
+
+    // The catalog is unchanged.
+    expect(await catalog(target!)).toEqual(await catalog(source!));
+
+    // Exporting the applied server gives back the same declarations.
+    const again = await exportResources({ environment: "rt", config: {}, env: env(target!) });
+    expect(new ClickHouseGenerator().generate(again)[0]!.content).toBe(file!.content);
+
+    // Every declared object is present: nothing to create.
+    const entities = new Map(
+      [...result.entities].map(([k, v]) => [k, { entityType: v.entityType, props: (v as unknown as { props: Record<string, unknown> }).props }]),
+    );
+    const observed = await describeResources({ environment: "rt", entityNames: [...entities.keys()], entities, config: {}, env: env(target!) });
+    expect(Object.keys(observed).length === 0 ? [] : Object.keys((observed as { resources?: object }).resources ?? observed).sort()).toEqual([...entities.keys()].sort());
+  }, 600_000);
+});

package/src/clickhouse/overlays/codecs.ts

@@ -0,0 +1,68 @@
+/**
+ * Codec parameters and chaining: the overlay half.
+ *
+ * `system.codecs` gives the 17 names and their flags (compression, generic,
+ * encryption, time series, experimental). The parameters (`ZSTD(level)`,
+ * `Delta(bytes)`) and the order a chain like `CODEC(Delta, ZSTD(3))` must
+ * follow are in no table.
+ *
+ * Every codec at the pin has an entry, and generation fails when one does not,
+ * so a codec a pin move adds is a decision rather than a silent `string`.
+ * `"unknown"` is that decision for the experimental codecs whose parameters
+ * are not documented: they are accepted as written and not checked.
+ */
+
+import type { ArgumentOverlay } from "./kinds";
+
+export interface CodecOverlay {
+  /** Positional parameters, all optional unless marked otherwise; `"unknown"` when undocumented. */
+  parameters: readonly (ArgumentOverlay & { name: string })[] | "unknown";
+  /**
+   * Where the codec sits in a chain. A `preprocessing` codec transforms values
+   * for the compression codec that must follow it; `encryption` comes last;
+   * `internal` is never written by hand.
+   */
+  role: "preprocessing" | "compression" | "encryption" | "none" | "internal";
+}
+
+const WIDTH: ArgumentOverlay & { name: string } = {
+  name: "bytes",
+  kind: "number",
+  optional: true,
+  note: "1, 2, 4 or 8; the column type's width when omitted.",
+};
+
+export const CODEC_OVERLAY: Record<string, CodecOverlay> = {
+  NONE: { parameters: [], role: "none" },
+  LZ4: { parameters: [], role: "compression" },
+  LZ4HC: {
+    parameters: [{ name: "level", kind: "number", optional: true, range: [0, 12], note: "9 when omitted or 0." }],
+    role: "compression",
+  },
+  ZSTD: {
+    parameters: [{ name: "level", kind: "number", optional: true, range: [1, 22], note: "1 when omitted." }],
+    role: "compression",
+  },
+  Delta: { parameters: [WIDTH], role: "preprocessing" },
+  DoubleDelta: { parameters: [WIDTH], role: "compression" },
+  Gorilla: { parameters: [WIDTH], role: "compression" },
+  GCD: { parameters: [], role: "preprocessing" },
+  T64: {
+    parameters: [{ name: "variant", kind: "keyword", optional: true, values: ["'byte'", "'bit'"] }],
+    role: "compression",
+  },
+  FPC: {
+    parameters: [
+      { name: "level", kind: "number", optional: true, range: [1, 28], note: "12 when omitted." },
+      { name: "float_size", kind: "number", optional: true, note: "4 or 8; the column type's width when omitted." },
+    ],
+    role: "compression",
+  },
+  AES_128_GCM_SIV: { parameters: [], role: "encryption" },
+  AES_256_GCM_SIV: { parameters: [], role: "encryption" },
+  ALP: { parameters: "unknown", role: "compression" },
+  SZ3: { parameters: "unknown", role: "compression" },
+  ZXC: { parameters: "unknown", role: "compression" },
+  Quantized: { parameters: "unknown", role: "preprocessing" },
+  Multiple: { parameters: "unknown", role: "internal" },
+};

package/src/clickhouse/overlays/engines.ts

@@ -0,0 +1,108 @@
+/**
+ * Engine arguments: the overlay half (chant #3195).
+ *
+ * The generator reads each engine's argument names, positions and optionality
+ * from the server's `syntax` line (`engine-syntax.ts`). This table adds what
+ * that line cannot say: whether `ver` is a column, `cluster` a name, `min_time`
+ * a number. Keys are the argument names exactly as the syntax line spells
+ * them, and generation fails when one no longer appears there, so a pin move
+ * that renames or drops an argument is caught rather than mistyped.
+ *
+ * Covered: the MergeTree family and its Replicated twins, and the engines a
+ * schema commonly declares beside them (Distributed, Buffer, Join, Merge,
+ * Dictionary, File, URL, Alias, KeeperMap, EmbeddedRocksDB, GenerateRandom).
+ * Integration engines (S3, Kafka, the lake engines, the database bridges) keep
+ * their arguments untyped: they are connection strings and credentials, which
+ * lint has no business guessing at.
+ */
+
+import type { ArgumentOverlay } from "./kinds";
+
+const REPLICATED: Record<string, ArgumentOverlay> = {
+  zoo_path: {
+    kind: "string",
+    optional: true,
+    note: "May be omitted together with replica_name when the server sets default_replica_path and default_replica_name.",
+  },
+  replica_name: { kind: "string", optional: true, note: "Omitted together with zoo_path." },
+};
+
+const SIGN: ArgumentOverlay = { kind: "column", note: "An Int8 column holding 1 for a state row and -1 for a cancel row." };
+const VERSION: ArgumentOverlay = { kind: "column", note: "An unsigned integer, date or datetime column." };
+const SUMMED: ArgumentOverlay = { kind: "columns", note: "Numeric columns to sum; every numeric column outside the sort key when omitted." };
+const REPLACING: Record<string, ArgumentOverlay> = {
+  ver: { kind: "column", note: "The version column: the row with the highest version wins; the last inserted when omitted." },
+  is_deleted: { kind: "column", note: "A UInt8 column; 1 marks a deleted row. Requires ver." },
+};
+
+export const ENGINE_ARGUMENTS: Record<string, Record<string, ArgumentOverlay>> = {
+  ReplacingMergeTree: REPLACING,
+  SummingMergeTree: { columns: SUMMED },
+  CoalescingMergeTree: { columns: SUMMED },
+  CollapsingMergeTree: { sign: SIGN },
+  VersionedCollapsingMergeTree: { sign: SIGN, version: VERSION },
+  GraphiteMergeTree: { config_section: { kind: "string", note: "The graphite_rollup section of the server config." } },
+
+  ReplicatedMergeTree: REPLICATED,
+  ReplicatedAggregatingMergeTree: REPLICATED,
+  ReplicatedReplacingMergeTree: { ...REPLICATED, ...REPLACING },
+  ReplicatedSummingMergeTree: { ...REPLICATED, columns: SUMMED },
+  ReplicatedCoalescingMergeTree: { ...REPLICATED, columns: SUMMED },
+  ReplicatedCollapsingMergeTree: { ...REPLICATED, sign: SIGN },
+  ReplicatedVersionedCollapsingMergeTree: { ...REPLICATED, sign: SIGN, version: VERSION },
+  ReplicatedGraphiteMergeTree: { ...REPLICATED, config_section: { kind: "string" } },
+
+  Distributed: {
+    cluster: { kind: "identifier", note: "A cluster named in the server's remote_servers config." },
+    database: { kind: "identifier" },
+    table: { kind: "identifier" },
+    sharding_key: { kind: "expression", note: "Required for INSERTs into a cluster of more than one shard." },
+    policy_name: { kind: "string", note: "A storage policy for the files queued for sending." },
+  },
+  Buffer: {
+    database: { kind: "identifier" },
+    table: { kind: "identifier" },
+    num_buckets: { kind: "number", range: [1, 1000], note: "Independent buffers; 16 is the documented default." },
+    min_time: { kind: "number" },
+    max_time: { kind: "number" },
+    min_rows: { kind: "number" },
+    max_rows: { kind: "number" },
+    min_bytes: { kind: "number" },
+    max_bytes: { kind: "number" },
+    flush_time: { kind: "number" },
+    flush_rows: { kind: "number" },
+    flush_bytes: { kind: "number" },
+  },
+  Join: {
+    join_strictness: { kind: "keyword", values: ["ANY", "ALL", "SEMI", "ANTI"] },
+    join_type: { kind: "keyword", values: ["INNER", "LEFT", "RIGHT", "FULL", "CROSS"] },
+    k1: { kind: "column" },
+    k2: { kind: "column" },
+  },
+  Merge: {
+    db_name: { kind: "expression", note: "A database name, or REGEXP('pattern')." },
+    tables_regexp: { kind: "string" },
+  },
+  Dictionary: { dictionary_name: { kind: "identifier" } },
+  File: {
+    format: { kind: "identifier", note: "An input/output format name from system.formats." },
+    "path|fd": { kind: "string" },
+  },
+  URL: {
+    url: { kind: "string" },
+    format: { kind: "identifier", note: "A format name from system.formats." },
+    compression: { kind: "string" },
+  },
+  Alias: { target_db: { kind: "identifier" }, target_table: { kind: "identifier" } },
+  KeeperMap: { root_path: { kind: "string" }, keys_limit: { kind: "number" } },
+  EmbeddedRocksDB: {
+    ttl: { kind: "number" },
+    rocksdb_dir: { kind: "string" },
+    read_only: { kind: "number", range: [0, 1] },
+  },
+  GenerateRandom: {
+    random_seed: { kind: "number" },
+    max_string_length: { kind: "number" },
+    max_array_length: { kind: "number" },
+  },
+};

package/src/clickhouse/overlays/kinds.ts

@@ -0,0 +1,38 @@
+/**
+ * What kind of value an argument or parameter takes. The catalog names an
+ * argument (`ver`, `cluster`); it never says whether `ver` is a column and
+ * `cluster` a name, which is what these say.
+ */
+export type ArgumentKind =
+  /** One column of the table, by name. */
+  | "column"
+  /** A column, or a tuple of columns. */
+  | "columns"
+  /** Any expression over the table's columns. */
+  | "expression"
+  /** A database, table, cluster or dictionary name. */
+  | "identifier"
+  /** A string literal. */
+  | "string"
+  /** A numeric literal. */
+  | "number"
+  /** A bare keyword from a fixed set (`values`). */
+  | "keyword"
+  /** A column type (`Nullable(T)`'s `T`). */
+  | "type"
+  /** An aggregate function name with its parameters (`AggregateFunction(uniq, UUID)`'s `uniq`). */
+  | "function"
+  /** A codec, as written inside `CODEC(...)`. */
+  | "codec";
+
+export interface ArgumentOverlay {
+  kind: ArgumentKind;
+  /** Overrides the syntax line's optionality, with `note` saying why. */
+  optional?: boolean;
+  /** The legal values of a `keyword`. */
+  values?: readonly string[];
+  /** The inclusive range of a `number`. */
+  range?: readonly [number, number];
+  /** Where the overlay knows something the syntax line does not. */
+  note?: string;
+}