@intentius/chant-lexicon-sql 0.121.1 → 0.122.0

package/src/clickhouse/overlays/codecs.ts

@@ -29,6 +29,7 @@ const WIDTH: ArgumentOverlay & { name: string } = {
   name: "bytes",
   kind: "number",
   optional: true,
+  values: ["1", "2", "4", "8"],
   note: "1, 2, 4 or 8; the column type's width when omitted.",
 };
 
@@ -40,7 +41,10 @@ export const CODEC_OVERLAY: Record<string, CodecOverlay> = {
     role: "compression",
   },
   ZSTD: {
-    parameters: [{ name: "level", kind: "number", optional: true, range: [1, 22], note: "1 when omitted." }],
+    parameters: [
+      { name: "level", kind: "number", optional: true, range: [1, 22], note: "1 when omitted." },
+      { name: "window_log", kind: "number", optional: true, range: [10, 31], note: "Turns on long-range matching with this window log." },
+    ],
     role: "compression",
   },
   Delta: { parameters: [WIDTH], role: "preprocessing" },
@@ -54,7 +58,7 @@ export const CODEC_OVERLAY: Record<string, CodecOverlay> = {
   FPC: {
     parameters: [
       { name: "level", kind: "number", optional: true, range: [1, 28], note: "12 when omitted." },
-      { name: "float_size", kind: "number", optional: true, note: "4 or 8; the column type's width when omitted." },
+      { name: "float_size", kind: "number", optional: true, values: ["4", "8"], note: "4 or 8; the column type's width when omitted." },
     ],
     role: "compression",
   },

package/src/clickhouse/overlays/kinds.ts

@@ -29,7 +29,7 @@ export interface ArgumentOverlay {
   kind: ArgumentKind;
   /** Overrides the syntax line's optionality, with `note` saying why. */
   optional?: boolean;
-  /** The legal values of a `keyword`. */
+  /** The legal values of a `keyword`, or of a `number` limited to a set (a codec's byte width: 1, 2, 4 or 8). */
   values?: readonly string[];
   /** The inclusive range of a `number`. */
   range?: readonly [number, number];

package/src/clickhouse/overlays/types.ts

@@ -26,16 +26,33 @@ const P = (name: string, kind: TypeParameter["kind"], extra: Partial<TypeParamet
 
 const TZ = P("timezone", "string", { optional: true });
 
+/**
+ * The server takes and ignores a MySQL-style display width on an integer
+ * (`INT(11)`, `UInt64(8)`), a precision and scale on a float (`DOUBLE(10, 2)`)
+ * and a length on a string (`VARCHAR(255)`), so each is an optional parameter
+ * here rather than a mistake.
+ */
+const WIDTH = [P("display_width", "number", { optional: true })];
+const FLOAT = [P("precision", "number", { optional: true }), P("scale", "number", { optional: true })];
+const INTEGERS = ["Int8", "Int16", "Int32", "Int64", "Int128", "Int256", "UInt8", "UInt16", "UInt32", "UInt64", "UInt128", "UInt256"];
+
 export const TYPE_PARAMETERS: Record<string, readonly TypeParameter[]> = {
-  Decimal: [P("precision", "number", { range: [1, 76] }), P("scale", "number", { optional: true })],
+  ...Object.fromEntries(INTEGERS.map((name) => [name, WIDTH])),
+  Float32: FLOAT,
+  Float64: FLOAT,
+  BFloat16: FLOAT,
+  String: [P("length", "number", { optional: true })],
+  /** A bare `Decimal` is `Decimal(10, 0)`. */
+  Decimal: [P("precision", "number", { optional: true, range: [1, 76] }), P("scale", "number", { optional: true, range: [0, 76] })],
   Decimal32: [P("scale", "number", { range: [0, 9] })],
   Decimal64: [P("scale", "number", { range: [0, 18] })],
   Decimal128: [P("scale", "number", { range: [0, 38] })],
   Decimal256: [P("scale", "number", { range: [0, 76] })],
   DateTime: [TZ],
   DateTime32: [TZ],
-  DateTime64: [P("precision", "number", { range: [0, 9] }), TZ],
-  Time64: [P("precision", "number", { range: [0, 9] })],
+  /** A bare `DateTime64` has precision 3. */
+  DateTime64: [P("precision", "number", { optional: true, range: [0, 9] }), TZ],
+  Time64: [P("precision", "number", { optional: true, range: [0, 9] })],
   FixedString: [P("length", "number", { range: [1, Number.MAX_SAFE_INTEGER] })],
   Enum: [P("member", "enum-member", { repeated: true })],
   Enum8: [P("member", "enum-member", { repeated: true })],

package/src/clickhouse/type-syntax.ts

@@ -0,0 +1,82 @@
+/**
+ * A column type, read into its family and parameters: `Nullable(String)` is
+ * the family `Nullable` with the one parameter `String`;
+ * `DateTime64(3, 'UTC')` is `DateTime64` with `3` and `'UTC'`.
+ *
+ * The reading is lexical, over the same tokenizer the parser uses, so a
+ * string literal holding a parenthesis (`Enum8('a(' = 1)`) or a comma does not
+ * split a parameter. What a parameter means (a type, a number, a name) is the
+ * caller's question, answered from `TYPE_PARAMETERS`.
+ */
+
+import { isTrivia, tokenizeText, type Token } from "./tokens";
+
+export interface TypeParam {
+  /** The parameter as written, trimmed. */
+  text: string;
+  tokens: Token[];
+}
+
+export interface TypeSyntax {
+  /** The words before the parameters: `UInt64`, `BIGINT UNSIGNED`, or `a String` for a named tuple element. */
+  words: Token[];
+  /** The parameters, or undefined when the type has no parentheses. */
+  params?: TypeParam[];
+}
+
+const textOf = (tokens: readonly Token[]): string => tokens.map((t) => t.text).join("").trim();
+
+/** The significant tokens of a type, or undefined when it does not tokenize. */
+export function typeTokens(text: string): Token[] | undefined {
+  try {
+    return tokenizeText(text, 0);
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Read a type into its leading words and its parameters. Undefined when the
+ * text is not a type this can read: no leading name, or unbalanced parentheses.
+ * Anything after the closing parenthesis (`COLLATE utf8`) is ignored.
+ */
+export function readType(tokens: readonly Token[]): TypeSyntax | undefined {
+  const sig = tokens.filter((t) => !isTrivia(t));
+  const words: Token[] = [];
+  let i = 0;
+  // `String COLLATE utf8`: COLLATE ends the type's words.
+  while (i < sig.length && (sig[i]!.kind === "ident" || sig[i]!.kind === "qident") && sig[i]!.text.toUpperCase() !== "COLLATE") words.push(sig[i++]!);
+  if (words.length === 0) return undefined;
+  if (i >= sig.length || !(sig[i]!.kind === "punct" && sig[i]!.text === "(")) return { words };
+  // Parameters: the tokens up to the matching parenthesis, split at top-level commas.
+  const start = tokens.indexOf(sig[i]!);
+  const params: TypeParam[] = [];
+  let depth = 0;
+  let current: Token[] = [];
+  for (let j = start; j < tokens.length; j++) {
+    const t = tokens[j]!;
+    if (t.kind === "punct" && t.text === "(") {
+      if (depth++ === 0) continue;
+    } else if (t.kind === "punct" && t.text === ")") {
+      if (--depth === 0) {
+        if (params.length > 0 || current.some((x) => !isTrivia(x))) params.push({ text: textOf(current), tokens: current });
+        return { words, params };
+      }
+    } else if (depth === 1 && t.kind === "punct" && t.text === ",") {
+      params.push({ text: textOf(current), tokens: current });
+      current = [];
+      continue;
+    }
+    current.push(t);
+  }
+  return undefined;
+}
+
+/** A parameter that is a numeric literal, possibly negative: its value. */
+export function numericParam(param: TypeParam): number | undefined {
+  const sig = param.tokens.filter((t) => !isTrivia(t));
+  const negative = sig.length === 2 && sig[0]!.kind === "op" && sig[0]!.text === "-";
+  const n = negative ? sig[1] : sig.length === 1 ? sig[0] : undefined;
+  if (!n || n.kind !== "number" || !/^\d+(\.\d+)?([eE][+-]?\d+)?$/.test(n.text)) return undefined;
+  return negative ? -Number(n.text) : Number(n.text);
+}

package/src/codegen/clickhouse-module.ts

@@ -219,7 +219,7 @@ export function renderClickHouseModule(catalog: ClickHouseCatalog): RenderedModu
 
   const tables = [
     ...header("The tables lint and the LSP read, typed by ./clickhouse-types."),
-    `import type { CodecSpec, DatabaseEngineSpec, SettingSpec, SkipIndexSpec, TableEngineSpec, TypeFamilySpec } from "../clickhouse/catalog-types";`,
+    `import type { CodecSpec, DatabaseEngineSpec, FunctionSpec, SettingSpec, SkipIndexSpec, TableEngineSpec, TypeFamilySpec } from "../clickhouse/catalog-types";`,
     `import type { CodecName, ColumnTypeName, DatabaseEngineName, MergeTreeSettings, QuerySettings, SkipIndexType, TableEngineName } from "./clickhouse-types";`,
     "",
     `export * from "./clickhouse-types";`,
@@ -255,6 +255,20 @@ export function renderClickHouseModule(catalog: ClickHouseCatalog): RenderedModu
     }),
     "};",
     "",
+    `/** Every function the server has (\`system.functions\`, origin System), aliases included. */`,
+    `export const FUNCTIONS: Readonly<Record<string, FunctionSpec>> = {`,
+    ...catalog.functions.map(
+      (f) =>
+        `  ${key(f.name)}: ${JSON.stringify({ aggregate: f.aggregate, caseInsensitive: f.caseInsensitive, ...(f.aliasOf ? { aliasOf: f.aliasOf } : {}) })},`,
+    ),
+    "};",
+    "",
+    `/** The aggregate function combinators (\`system.aggregate_function_combinators\`): the suffixes of \`countIf\`, \`sumState\`. */`,
+    `export const AGGREGATE_COMBINATORS: readonly string[] = ${JSON.stringify(catalog.aggregateCombinators.map((c) => c.name))};`,
+    "",
+    `/** The table functions (\`system.table_functions\`): \`numbers\`, \`remote\`, \`s3\`. */`,
+    `export const TABLE_FUNCTIONS: readonly string[] = ${JSON.stringify(catalog.tableFunctions)};`,
+    "",
     `export const SKIP_INDEX_TYPES: Readonly<Record<SkipIndexType, SkipIndexSpec>> = {`,
     ...indexLines,
     "};",

package/src/codegen/postgres-module.ts

@@ -410,6 +410,43 @@ export function renderPostgresModule(catalogs: readonly PostgresCatalog[]): Rend
     "};",
   ];
 
+  // Names lint checks a declaration against, each with the majors that have it
+  // (`true`: every supported major). Read straight off the catalogs, internal
+  // functions included: a column default may call `pg_current_xact_id()`.
+  const presence = (lists: ReadonlyArray<readonly string[]>): Map<string, number[]> => {
+    const out = new Map<string, number[]>();
+    lists.forEach((names, i) => {
+      for (const n of new Set(names)) out.set(n, [...(out.get(n) ?? []), majors[i]!]);
+    });
+    return out;
+  };
+  const majorsLiteral = (ms: readonly number[]): string => (ms.length === majors.length ? "true" : JSON.stringify(ms));
+  const presenceTable = (m: Map<string, number[]>, indent: string): string[] =>
+    sorted(m.keys()).map((n) => `${indent}${JSON.stringify(n)}: ${majorsLiteral(m.get(n)!)},`);
+  const functionTable = [
+    `/** Every function, aggregate and procedure name in \`pg_proc\`, to the majors that have it (\`true\`: all of them). */`,
+    `export const FUNCTIONS: Readonly<Record<string, true | readonly number[]>> = {`,
+    ...presenceTable(presence(catalogs.map((c) => c.functions.map((f) => f.name))), "  "),
+    "};",
+  ];
+  const methodTable = [
+    `/** The index access methods, to the majors that have each (\`true\`: all of them). */`,
+    `export const INDEX_ACCESS_METHODS: Readonly<Record<string, true | readonly number[]>> = {`,
+    ...presenceTable(presence(catalogs.map((c) => c.accessMethods.filter((a) => a.type === "i").map((a) => a.name))), "  "),
+    "};",
+  ];
+  const opclassAms = sorted(catalogs.flatMap((c) => c.opclasses.map((o) => o.am)));
+  const opclassTable = [
+    `/** The operator classes of each index access method, to the majors that have each (\`true\`: all of them). */`,
+    `export const OPERATOR_CLASSES: Readonly<Record<string, Readonly<Record<string, true | readonly number[]>>>> = {`,
+    ...opclassAms.flatMap((am) => [
+      `  ${key(am)}: {`,
+      ...presenceTable(presence(catalogs.map((c) => c.opclasses.filter((o) => o.am === am).map((o) => o.name))), "    "),
+      "  },",
+    ]),
+    "};",
+  ];
+
   const tables = [
     ...header("The tables lint and the LSP read, typed by ./postgres-types."),
     `import type { ColumnTypeSpec, SettingSpec, StorageParameterSpec } from "../postgres/catalog-types";`,
@@ -425,6 +462,12 @@ export function renderPostgresModule(catalogs: readonly PostgresCatalog[]): Rend
     "",
     ...keywordTable,
     "",
+    ...functionTable,
+    "",
+    ...methodTable,
+    "",
+    ...opclassTable,
+    "",
   ].join("\n");
 
   notes.push(

package/src/composites/composites.test.ts

@@ -202,7 +202,7 @@ test("a prop that is not valid SQL is refused at the interpolation that carried
 // ── What the lexicon's own checks say ──────────────────────────────────
 
 describe("the composites pass the lexicon's post-synth checks", () => {
-  test("SQLCH101-120 find nothing in any of them", () => {
+  test("SQL101 and SQLCH101-127 find nothing in any of them", () => {
     const source = events();
     const instances: Array<[string, CompositeInstance]> = [
       ["users", ReplacingTable({ name: "users", columns: "id UInt64, email String", orderBy: "id" })],
@@ -230,10 +230,11 @@ describe("the composites pass the lexicon's post-synth checks", () => {
     expect(doc.applyOrder.indexOf("hitsLocal")).toBeLessThan(doc.applyOrder.indexOf("hitsDistributed"));
 
     const ctx = makePostSynthCtx("sql", out.primary, entities);
-    const clickhouse = postSynthChecks.filter((c) => c.id.startsWith("SQLCH"));
+    const clickhouse = postSynthChecks.filter((c) => c.id.startsWith("SQLCH") || c.id === "SQL101");
     const ids = clickhouse.map((c) => c.id).sort();
-    expect(ids[0]).toBe("SQLCH101");
-    expect(ids.at(-1)).toBe("SQLCH120");
+    expect(ids[0]).toBe("SQL101");
+    expect(ids).toContain("SQLCH101");
+    expect(ids.at(-1)).toBe("SQLCH127");
     expect(clickhouse.flatMap((check) => check.check(ctx))).toEqual([]);
   });
 });
@@ -559,7 +560,7 @@ describe("the Postgres composites", () => {
     expect(() => TenantTable({ name: "notes", columns: "id bigint NOT NULL, (", primaryKey: "id", indexOn: "id" })).toThrow(SqlTemplateError);
   });
 
-  test("SQLPG101 to SQLPG118 find nothing in any of them", () => {
+  test("SQLPG101 to SQLPG126 find nothing in any of them", () => {
     const people = users();
     const teams = SoftDeleteTable({ name: "teams", schema: "app", columns: "id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, name text NOT NULL" });
     const members = JoinTable({ name: "memberships", schema: "app", left: people.table, leftColumn: "user_id", right: teams.table, rightColumn: "team_id" });
@@ -588,7 +589,7 @@ describe("the Postgres composites", () => {
 
     const ctx = makePostSynthCtx("sql", out.primary, entities);
     const checks = postSynthChecks.filter((c) => c.id.startsWith("SQLPG"));
-    expect(checks).toHaveLength(18);
+    expect(checks).toHaveLength(26);
     expect(checks.flatMap((check) => check.check(ctx).map((d) => `${check.id} ${d.message}`))).toEqual([]);
   });
 });