@intentius/chant-lexicon-sql 0.0.0-stage → 0.112.1

package/src/clickhouse/engine-syntax.test.ts

@@ -0,0 +1,53 @@
+import { describe, expect, test } from "vitest";
+import { parseEngineSyntax } from "./engine-syntax";
+
+const names = (syntax: string) =>
+  parseEngineSyntax(syntax)?.args.map((a) => `${a.optional ? "?" : ""}${a.name}${a.repeated ? "..." : ""}${a.named ? "=" : ""}`);
+
+describe("reading an engine's arguments from its syntax line", () => {
+  test("nested optional brackets mark every argument inside them optional", () => {
+    expect(names("ENGINE = ReplacingMergeTree([ver [, is_deleted]]) ORDER BY expr")).toEqual(["?ver", "?is_deleted"]);
+  });
+
+  test("an empty list and no list are different shapes", () => {
+    expect(parseEngineSyntax("ENGINE = MergeTree() ORDER BY expr")).toEqual({ name: "MergeTree", parenthesised: true, args: [] });
+    expect(parseEngineSyntax("ENGINE = Log")).toEqual({ name: "Log", parenthesised: false, args: [] });
+  });
+
+  test("a quoted placeholder is a string literal argument", () => {
+    const parsed = parseEngineSyntax("ENGINE = ReplicatedMergeTree('zoo_path', 'replica_name') ORDER BY expr");
+    expect(parsed?.args.map((a) => [a.name, a.quoted])).toEqual([
+      ["zoo_path", true],
+      ["replica_name", true],
+    ]);
+  });
+
+  test("alternatives become one argument, and a trailing ... repeats the last", () => {
+    expect(names("ENGINE = File(format[, path | fd])")).toEqual(["format", "?path|fd"]);
+    expect(names("ENGINE = Join(join_strictness, join_type, k1[, k2, ...])")).toEqual([
+      "join_strictness",
+      "join_type",
+      "k1",
+      "?k2...",
+    ]);
+  });
+
+  test("name = value is a named parameter", () => {
+    expect(names("INDEX name expr TYPE text(tokenizer = splitByNonAlpha) GRANULARITY g")).toEqual(["tokenizer="]);
+  });
+
+  test("a skip index line is read the same way as an engine line", () => {
+    expect(names("INDEX name expr TYPE ngrambf_v1(n, size_in_bytes, num_hash_functions, seed) GRANULARITY g")).toEqual([
+      "n",
+      "size_in_bytes",
+      "num_hash_functions",
+      "seed",
+    ]);
+  });
+
+  test("a line that is not an ENGINE or INDEX line is not guessed at", () => {
+    expect(parseEngineSyntax("CREATE MATERIALIZED VIEW name [TO target] AS SELECT ...")).toBeUndefined();
+    expect(parseEngineSyntax("")).toBeUndefined();
+    expect(parseEngineSyntax("ENGINE = Broken(a]")).toBeUndefined();
+  });
+});

package/src/clickhouse/engine-syntax.ts

@@ -0,0 +1,173 @@
+/**
+ * Read the argument list out of a ClickHouse engine's one-line `syntax`
+ * (`system.table_engines.syntax`, `system.data_skipping_index_types.syntax`).
+ *
+ * The line is prose, not grammar: `ReplacingMergeTree([ver [, is_deleted]])`,
+ * `Join(join_strictness, join_type, k1[, k2, ...])`,
+ * `File(format[, path | fd])`. What it does give reliably is each argument's
+ * name, position, whether it sits inside brackets (optional), whether it is a
+ * quoted placeholder (a string literal), and a trailing `...` (repeatable).
+ * That is the generated half of an engine's argument types; what kind of value
+ * each argument takes is the overlay's half (`overlays/engines.ts`).
+ *
+ * A line this reader cannot follow yields `undefined`, never a guess, and the
+ * engine's arguments stay untyped.
+ */
+
+export interface SyntaxArgument {
+  /** The placeholder's name. Alternatives are joined with `|`: `path|fd`. A quoted placeholder keeps its text: `host:port`. */
+  name: string;
+  /** 0-based position among the arguments. */
+  position: number;
+  /** Inside `[...]` in the syntax line. */
+  optional: boolean;
+  /** Written as a quoted placeholder, so the argument is a string literal. */
+  quoted: boolean;
+  /** Followed by `...`: the argument repeats. */
+  repeated: boolean;
+  /** Written `name = value`: a named parameter, not a positional one (`text(tokenizer = ...)`). */
+  named: boolean;
+}
+
+export interface ParsedSyntax {
+  /** The engine or index type name the line opens with. */
+  name: string;
+  /** False when the line has no parenthesised list at all (`ENGINE = Log`). */
+  parenthesised: boolean;
+  args: SyntaxArgument[];
+}
+
+type Token =
+  | { t: "name"; v: string }
+  | { t: "quoted"; v: string }
+  | { t: "[" | "]" | "," | "|" | "..." | "=" };
+
+function tokenize(text: string): Token[] | undefined {
+  const out: Token[] = [];
+  let i = 0;
+  while (i < text.length) {
+    const c = text[i]!;
+    if (/\s/.test(c)) {
+      i++;
+    } else if (c === "[" || c === "]" || c === "," || c === "|" || c === "=") {
+      out.push({ t: c });
+      i++;
+    } else if (text.startsWith("...", i)) {
+      out.push({ t: "..." });
+      i += 3;
+    } else if (c === "'") {
+      const end = text.indexOf("'", i + 1);
+      if (end < 0) return undefined;
+      out.push({ t: "quoted", v: text.slice(i + 1, end) });
+      i = end + 1;
+    } else {
+      const m = /^[A-Za-z_][\w.]*/.exec(text.slice(i));
+      if (!m) return undefined;
+      out.push({ t: "name", v: m[0] });
+      i += m[0].length;
+    }
+  }
+  return out;
+}
+
+/** The text between the first `(` after the name and its matching `)`. */
+function argumentText(rest: string): string | undefined {
+  if (!rest.startsWith("(")) return undefined;
+  let depth = 0;
+  for (let i = 0; i < rest.length; i++) {
+    if (rest[i] === "(") depth++;
+    else if (rest[i] === ")") {
+      depth--;
+      if (depth === 0) return rest.slice(1, i);
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Parse `ENGINE = Name(args) ...` or `INDEX name expr TYPE Name(args) ...`.
+ * Returns `undefined` for a line that is not one of those shapes or whose
+ * argument list does not follow the bracket-and-comma convention.
+ */
+export function parseEngineSyntax(syntax: string): ParsedSyntax | undefined {
+  const head = /^(?:ENGINE\s*=\s*|INDEX\s+\S+\s+\S+\s+TYPE\s+)([A-Za-z_]\w*)/.exec(syntax.trim());
+  if (!head) return undefined;
+  const name = head[1]!;
+  const rest = syntax.trim().slice(head[0].length);
+  if (!rest.startsWith("(")) return { name, parenthesised: false, args: [] };
+  const inner = argumentText(rest);
+  if (inner === undefined) return undefined;
+  const tokens = tokenize(inner);
+  if (!tokens) return undefined;
+
+  const args: SyntaxArgument[] = [];
+  let depth = 0;
+  let pending: { names: string[]; quoted: boolean; optional: boolean; named: boolean } | undefined;
+  let afterBar = false;
+  let afterEquals = false;
+
+  const flush = () => {
+    if (!pending) return;
+    args.push({
+      name: pending.names.join("|"),
+      position: args.length,
+      optional: pending.optional,
+      quoted: pending.quoted,
+      repeated: false,
+      named: pending.named,
+    });
+    pending = undefined;
+  };
+
+  for (const tok of tokens) {
+    switch (tok.t) {
+      case "[":
+        depth++;
+        break;
+      case "]":
+        if (depth === 0) return undefined;
+        depth--;
+        break;
+      case ",":
+        flush();
+        afterBar = false;
+        afterEquals = false;
+        break;
+      case "|":
+        if (!pending) return undefined;
+        afterBar = true;
+        break;
+      case "...":
+        // `input_query...` repeats the argument it follows; `k2, ...` repeats the last one.
+        flush();
+        if (args.length === 0) return undefined;
+        args[args.length - 1]!.repeated = true;
+        break;
+      case "=":
+        // `key = value, ...` is a settings tail (BigQuery); `tokenizer = x` names a parameter (text index).
+        if (!pending || pending.names.length !== 1) return undefined;
+        pending.named = true;
+        afterEquals = true;
+        break;
+      case "name":
+      case "quoted": {
+        if (pending && afterEquals) {
+          // The value placeholder after `=` is an example, not a second argument.
+          afterEquals = false;
+          break;
+        }
+        if (pending && afterBar) {
+          pending.names.push(tok.v);
+          afterBar = false;
+          break;
+        }
+        if (pending) return undefined;
+        pending = { names: [tok.v], quoted: tok.t === "quoted", optional: depth > 0, named: false };
+        break;
+      }
+    }
+  }
+  flush();
+  if (depth !== 0) return undefined;
+  return { name, parenthesised: true, args };
+}

package/src/clickhouse/entities.test.ts

@@ -0,0 +1,188 @@
+import { describe, expect, test } from "vitest";
+import { isDeclarable } from "@intentius/chant/declarable";
+import { buildDependencyGraph } from "@intentius/chant/discovery/graph";
+import { getInterpolationFields } from "@intentius/chant/provenance";
+import { database, isColumnRef, literal, table, view, SqlTemplateError } from "./entities";
+
+const users = table`
+  CREATE TABLE users (
+    id     UUID,
+    email  String,
+    plan   LowCardinality(String)
+  )
+  ENGINE = ReplacingMergeTree
+  ORDER BY id`;
+
+const events = table`
+  CREATE TABLE events (
+    user_id  UUID,
+    kind     LowCardinality(String),
+    ts       DateTime
+  )
+  ENGINE = MergeTree
+  ORDER BY (user_id, kind, ts)`;
+
+const activeUsers = view`
+  CREATE MATERIALIZED VIEW active_users
+  ENGINE = AggregatingMergeTree ORDER BY ${events.columns.user_id} AS
+  SELECT ${events.columns.user_id}, count() AS n
+  FROM ${events}
+  GROUP BY ${events.columns.user_id}`;
+
+describe("the #3047 example", () => {
+  test("each tag is one declarable entity of its ClickHouse type", () => {
+    expect(isDeclarable(users)).toBe(true);
+    expect(users.entityType).toBe("ClickHouse::Table");
+    expect(activeUsers.entityType).toBe("ClickHouse::MaterializedView");
+    expect(users.lexicon).toBe("sql");
+  });
+
+  test("a table's props are its parsed definition", () => {
+    expect(events.props.name).toBe("events");
+    expect(events.props.columns.map((c) => [c.name, c.type])).toEqual([
+      ["user_id", "UUID"],
+      ["kind", "LowCardinality(String)"],
+      ["ts", "DateTime"],
+    ]);
+    expect(events.props.engine).toEqual({ name: "MergeTree" });
+    expect(events.props.orderBy).toBe("(user_id, kind, ts)");
+  });
+
+  test("columns are references, reached through .columns", () => {
+    expect(isColumnRef(events.columns.kind)).toBe(true);
+    expect(events.columns.kind!.attribute).toBe("kind");
+    // `kind` on the entity itself is the Declarable's own field, which is why columns live under `.columns`.
+    expect(events.kind).toBe("resource");
+  });
+
+  test("a view records what it reads and its column lineage", () => {
+    expect(activeUsers.props.reads).toEqual([events]);
+    expect(activeUsers.props.lineage.map((e) => ({ output: e.output, expr: e.expr, from: e.from.map((r) => r.attribute) }))).toEqual([
+      { output: "user_id", expr: "user_id", from: ["user_id"] },
+      { output: "n", expr: "count()", from: [] },
+    ]);
+    expect(Object.keys(activeUsers.columns)).toEqual(["user_id", "n"]);
+  });
+
+  test("references render into the DDL as names", () => {
+    expect(activeUsers.props.ddl).toContain("ORDER BY user_id AS");
+    expect(activeUsers.props.ddl).toContain("FROM events");
+  });
+
+  test("the references are dependency edges core's graph sees", () => {
+    const graph = buildDependencyGraph(new Map<string, never>([["users", users as never], ["events", events as never], ["activeUsers", activeUsers as never]]));
+    expect([...graph.get("activeUsers")!]).toEqual(["events"]);
+    expect([...graph.get("events")!]).toEqual([]);
+  });
+
+  test("the raw template parts are kept for a source round trip", () => {
+    expect(users.props.source.strings.join("")).toContain("ReplacingMergeTree");
+  });
+});
+
+describe("what an interpolation means", () => {
+  test("a string is SQL text, spliced before parsing", () => {
+    const name = "orders";
+    const engine = "ReplacingMergeTree(version)";
+    const t = table`CREATE TABLE ${name} (id UInt64, version UInt32) ENGINE = ${engine} ORDER BY id`;
+    expect(t.props.name).toBe("orders");
+    expect(t.props.engine).toEqual({ name: "ReplacingMergeTree", args: ["version"] });
+  });
+
+  test("a number is a numeric literal", () => {
+    const days = 30;
+    const t = table`CREATE TABLE t (ts DateTime) ENGINE = MergeTree ORDER BY ts TTL ts + INTERVAL ${days} DAY`;
+    expect(t.props.ttl).toBe("ts + INTERVAL 30 DAY");
+  });
+
+  test("literal() is a quoted, escaped string value", () => {
+    const plan = "free's";
+    const t = table`CREATE TABLE t (plan String DEFAULT ${literal(plan)}) ENGINE = Log`;
+    expect(t.props.columns[0]!.default).toEqual({ kind: "DEFAULT", expr: "'free\\'s'" });
+  });
+
+  test("a database entity qualifies a table's name", () => {
+    const analytics = database`CREATE DATABASE analytics ENGINE = Atomic`;
+    const t = table`CREATE TABLE ${analytics}.events (a UInt8) ENGINE = Log`;
+    expect(t.props.database).toBe("analytics");
+    expect(t.sqlName).toBe("analytics.events");
+    expect(t.dependsOn).toEqual([analytics]);
+  });
+
+  test("a materialized view's TO target is the entity when it is declared", () => {
+    const dst = table`CREATE TABLE dst (n UInt64) ENGINE = SummingMergeTree ORDER BY tuple()`;
+    const mv = view`CREATE MATERIALIZED VIEW mv TO ${dst} AS SELECT count() AS n FROM ${events}`;
+    expect(mv.props.to).toBe(dst);
+  });
+
+  test("undefined is refused, naming .columns", () => {
+    const e = events as unknown as Record<string, unknown>;
+    expect(() => view`CREATE VIEW v AS SELECT ${e.user_id} FROM ${events}`).toThrow(/\.columns/);
+  });
+
+  test("an object is refused with the template line", () => {
+    expect(() => table`CREATE TABLE t (
+      a UInt8 DEFAULT ${{}}) ENGINE = Log`).toThrow(/template line 2/);
+  });
+
+  test("text a string splices in is parsed, and its errors name the interpolation", () => {
+    const bad = "Strin g";
+    expect(() => table`CREATE TABLE t (a ${bad}) ENGINE = Log`).toThrow(/interpolated on template line 1/);
+  });
+});
+
+describe("refusals", () => {
+  test("a tag holding another statement names the right tag", () => {
+    expect(() => table`CREATE VIEW v AS SELECT 1`).toThrow(/use the view tag/);
+  });
+
+  test("a syntax error says which template line it is on", () => {
+    try {
+      table`CREATE TABLE t (
+        a UInt8,
+        b Strin g
+      ) ENGINE = Log`;
+      expect.unreachable();
+    } catch (err) {
+      expect(err).toBeInstanceOf(SqlTemplateError);
+      expect((err as Error).message).toMatch(/template line 3/);
+    }
+  });
+
+  test("a backslash is read raw, as written", () => {
+    const t = table`CREATE TABLE t (re String DEFAULT '\d+\t') ENGINE = Log`;
+    expect(t.props.columns[0]!.default!.expr).toBe("'\\d+\\t'");
+  });
+});
+
+// #3212: the tag says which fields each interpolation fed, so a composite's
+// parameter can be traced to the field it produced.
+describe("the fields each interpolation fed", () => {
+  test("spliced text feeds the fields it was parsed into, and the DDL", () => {
+    const name = "app.clicks";
+    const t = table`
+      CREATE TABLE ${name} (${"url String"}, ts DateTime)
+      ENGINE = ReplacingMergeTree(${"ts"})
+      PARTITION BY ${"toYYYYMM(ts)"}
+      ORDER BY url
+      TTL ts + INTERVAL ${30} DAY
+      SETTINGS ${"index_granularity = 4096"}
+      COMMENT ${literal("hits")}`;
+    expect(getInterpolationFields(t)).toEqual([
+      ["database", "ddl", "name"],
+      ["columns", "ddl"],
+      ["ddl", "engine.args"],
+      ["ddl", "partitionBy"],
+      ["ddl", "ttl"],
+      ["ddl", "settings"],
+      ["comment", "ddl"],
+    ]);
+  });
+
+  test("an interpolated entity or column is a reference and feeds nothing", () => {
+    const v = view`
+      CREATE MATERIALIZED VIEW ${"by_kind"} ENGINE = SummingMergeTree ORDER BY ${events.columns.kind}
+      AS SELECT ${events.columns.kind}, count() AS n FROM ${events} GROUP BY ${events.columns.kind}`;
+    expect(getInterpolationFields(v)).toEqual([["ddl", "name"], [], [], [], []]);
+  });
+});