@intentius/chant-lexicon-sql 0.124.31 → 0.124.33

package/dist/clickhouse/container.js

@@ -0,0 +1,152 @@
+/**
+ * A throwaway `clickhouse-server` in a container: started on a random loopback
+ * port under a unique name, and always removed (the shared core's
+ * `../core/container.ts`, ready when it answers `/ping`).
+ *
+ * Generation uses it on a pin move to read the pinned server's catalog. The
+ * live tests use the same helper so every container chant starts is started
+ * and cleaned up one way. A cluster of replicas with embedded Keeper is
+ * ClickHouse's own.
+ */
+import { execFile } from "node:child_process";
+import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { clickhousePing, clickhouseQuery } from "./http.js";
+import { publishedPort, startScratchContainer, uniqueContainerName } from "../core/container.js";
+export { dockerAvailable } from "../core/container.js";
+const exec = promisify(execFile);
+/**
+ * Start `image` and wait until it answers `/ping`. Pulls the image when it is
+ * not present, which is the one network step here.
+ */
+export async function startScratchServer(image, options = {}) {
+    const endpointOf = (port) => ({ url: `http://127.0.0.1:${port}` });
+    const c = await startScratchContainer({
+        image,
+        namePrefix: options.namePrefix ?? "chant-sql",
+        containerPort: 8123,
+        env: { CLICKHOUSE_SKIP_USER_SETUP: "1" },
+        ready: (port) => clickhousePing(endpointOf(port)),
+        readyTimeoutMs: options.readyTimeoutMs ?? 60000,
+        notReady: "did not answer /ping",
+    });
+    return { endpoint: endpointOf(c.port), name: c.name, stop: c.stop };
+}
+/**
+ * Start servers of `image` on their own Docker network, and wait until each
+ * answers and reaches Keeper: `replicas` servers as one shard of replicas,
+ * or, with `shards`, that many replicas in each shard (`[2, 1]` is a shard of
+ * two replicas and a shard of one server).
+ *
+ * Keeper is the server's embedded one (`keeper_server`), run by the first
+ * server, so the pinned server image and its digest pin Keeper too. Each
+ * server gets the macros `{shard}` = `s<n>` and `{replica}` = `r<n>` (the
+ * replica numbered within its shard), which a `Replicated` database's and a
+ * `Replicated*MergeTree`'s arguments name, and `remote_servers` defines the
+ * cluster `cluster` (default `chant`) over all of them, with
+ * `internal_replication`, so `ON CLUSTER` reaches every server. Configuration
+ * is copied into each container before it starts, with no bind mount.
+ */
+export async function startScratchCluster(image, options = {}) {
+    const shards = options.shards ?? [options.replicas ?? 2];
+    const clusterName = options.cluster ?? "chant";
+    const name = uniqueContainerName(options.namePrefix ?? "chant-sql-cluster");
+    const network = `${name}-net`;
+    // One shard keeps the names the single-shard cluster always had (`<name>-r<n>`).
+    const layout = shards.flatMap((replicas, s) => Array.from({ length: replicas }, (_, r) => ({
+        shard: `s${s + 1}`,
+        replica: `r${r + 1}`,
+        host: shards.length === 1 ? `${name}-r${r + 1}` : `${name}-s${s + 1}r${r + 1}`,
+    })));
+    const nodes = layout.map((l) => l.host);
+    const dir = mkdtempSync(join(tmpdir(), "chant-sql-cluster-"));
+    let removed = false;
+    const stop = async () => {
+        if (removed)
+            return;
+        removed = true;
+        await exec("docker", ["rm", "-f", ...nodes]).catch(() => undefined);
+        await exec("docker", ["network", "rm", network]).catch(() => undefined);
+        rmSync(dir, { recursive: true, force: true });
+    };
+    const keeper = nodes[0];
+    const remoteServers = shards
+        .map((_, s) => {
+        const hosts = layout.filter((l) => l.shard === `s${s + 1}`).map((l) => `<replica><host>${l.host}</host><port>9000</port></replica>`);
+        return `<shard><internal_replication>true</internal_replication>${hosts.join("")}</shard>`;
+    })
+        .join("");
+    const config = (node, i) => `<clickhouse>
+${i === 0
+        ? `  <keeper_server>
+    <tcp_port>9181</tcp_port>
+    <server_id>1</server_id>
+    <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+    <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+    <raft_configuration><server><id>1</id><hostname>${keeper}</hostname><port>9234</port></server></raft_configuration>
+  </keeper_server>
+`
+        : ""}  <zookeeper><node><host>${keeper}</host><port>9181</port></node></zookeeper>
+  <distributed_ddl><path>/clickhouse/task_queue/ddl</path></distributed_ddl>
+  <remote_servers><${clusterName}>${remoteServers}</${clusterName}></remote_servers>
+  <macros><shard>${layout[i].shard}</shard><replica>${layout[i].replica}</replica></macros>
+  <interserver_http_host>${node}</interserver_http_host>
+</clickhouse>
+`;
+    const hostEndpoint = async (node) => ({ url: `http://127.0.0.1:${await publishedPort(node, 8123)}` });
+    const waitReady = async (node, endpoint, deadline) => {
+        for (;;) {
+            const ready = (await clickhousePing(endpoint)) && (await clickhouseQuery(endpoint, "SELECT count() FROM system.zookeeper WHERE path = '/'").then(() => true, () => false));
+            if (ready)
+                return;
+            if (Date.now() > deadline)
+                throw new Error(`${node} did not answer and reach Keeper within the timeout`);
+            await new Promise((r) => setTimeout(r, 500));
+        }
+    };
+    try {
+        await exec("docker", ["network", "create", network]);
+        const endpoints = await Promise.all(nodes.map(async (node, i) => {
+            await exec("docker", ["create", "--name", node, "--hostname", node, "--network", network, "-p", "127.0.0.1::8123", "-e", "CLICKHOUSE_SKIP_USER_SETUP=1", image], { timeout: 600000 });
+            const file = join(dir, `${node}.xml`);
+            writeFileSync(file, config(node, i));
+            // Readable by the server's own user once copied in, whatever the umask made it.
+            chmodSync(file, 0o644);
+            await exec("docker", ["cp", file, `${node}:/etc/clickhouse-server/config.d/chant-cluster.xml`]);
+            await exec("docker", ["start", node]);
+            return hostEndpoint(node);
+        }));
+        const deadline = Date.now() + (options.readyTimeoutMs ?? 90000);
+        for (const [i, endpoint] of endpoints.entries())
+            await waitReady(nodes[i], endpoint, deadline);
+        const node = (index) => {
+            const n = nodes[index];
+            if (!n)
+                throw new Error(`no replica ${index} in ${name}`);
+            return n;
+        };
+        return {
+            replicas: endpoints,
+            layout,
+            cluster: clusterName,
+            name,
+            stop,
+            async stopReplica(index) {
+                await exec("docker", ["stop", node(index)], { timeout: 120000 });
+            },
+            async startReplica(index) {
+                await exec("docker", ["start", node(index)]);
+                const endpoint = await hostEndpoint(node(index));
+                await waitReady(node(index), endpoint, Date.now() + (options.readyTimeoutMs ?? 90000));
+                endpoints[index] = endpoint;
+                return endpoint;
+            },
+        };
+    }
+    catch (err) {
+        await stop();
+        throw err;
+    }
+}

package/dist/clickhouse/engine-syntax.js

@@ -0,0 +1,159 @@
+/**
+ * Read the argument list out of a ClickHouse engine's one-line `syntax`
+ * (`system.table_engines.syntax`, `system.data_skipping_index_types.syntax`).
+ *
+ * The line is prose, not grammar: `ReplacingMergeTree([ver [, is_deleted]])`,
+ * `Join(join_strictness, join_type, k1[, k2, ...])`,
+ * `File(format[, path | fd])`. What it does give reliably is each argument's
+ * name, position, whether it sits inside brackets (optional), whether it is a
+ * quoted placeholder (a string literal), and a trailing `...` (repeatable).
+ * That is the generated half of an engine's argument types; what kind of value
+ * each argument takes is the overlay's half (`overlays/engines.ts`).
+ *
+ * A line this reader cannot follow yields `undefined`, never a guess, and the
+ * engine's arguments stay untyped.
+ */
+function tokenize(text) {
+    const out = [];
+    let i = 0;
+    while (i < text.length) {
+        const c = text[i];
+        if (/\s/.test(c)) {
+            i++;
+        }
+        else if (c === "[" || c === "]" || c === "," || c === "|" || c === "=") {
+            out.push({ t: c });
+            i++;
+        }
+        else if (text.startsWith("...", i)) {
+            out.push({ t: "..." });
+            i += 3;
+        }
+        else if (c === "'") {
+            const end = text.indexOf("'", i + 1);
+            if (end < 0)
+                return undefined;
+            out.push({ t: "quoted", v: text.slice(i + 1, end) });
+            i = end + 1;
+        }
+        else {
+            const m = /^[A-Za-z_][\w.]*/.exec(text.slice(i));
+            if (!m)
+                return undefined;
+            out.push({ t: "name", v: m[0] });
+            i += m[0].length;
+        }
+    }
+    return out;
+}
+/** The text between the first `(` after the name and its matching `)`. */
+function argumentText(rest) {
+    if (!rest.startsWith("("))
+        return undefined;
+    let depth = 0;
+    for (let i = 0; i < rest.length; i++) {
+        if (rest[i] === "(")
+            depth++;
+        else if (rest[i] === ")") {
+            depth--;
+            if (depth === 0)
+                return rest.slice(1, i);
+        }
+    }
+    return undefined;
+}
+/**
+ * Parse `ENGINE = Name(args) ...` or `INDEX name expr TYPE Name(args) ...`.
+ * Returns `undefined` for a line that is not one of those shapes or whose
+ * argument list does not follow the bracket-and-comma convention.
+ */
+export function parseEngineSyntax(syntax) {
+    const head = /^(?:ENGINE\s*=\s*|INDEX\s+\S+\s+\S+\s+TYPE\s+)([A-Za-z_]\w*)/.exec(syntax.trim());
+    if (!head)
+        return undefined;
+    const name = head[1];
+    const rest = syntax.trim().slice(head[0].length);
+    if (!rest.startsWith("("))
+        return { name, parenthesised: false, args: [] };
+    const inner = argumentText(rest);
+    if (inner === undefined)
+        return undefined;
+    const tokens = tokenize(inner);
+    if (!tokens)
+        return undefined;
+    const args = [];
+    let depth = 0;
+    let pending;
+    let afterBar = false;
+    let afterEquals = false;
+    const flush = () => {
+        if (!pending)
+            return;
+        args.push({
+            name: pending.names.join("|"),
+            position: args.length,
+            optional: pending.optional,
+            quoted: pending.quoted,
+            repeated: false,
+            named: pending.named,
+        });
+        pending = undefined;
+    };
+    for (const tok of tokens) {
+        switch (tok.t) {
+            case "[":
+                depth++;
+                break;
+            case "]":
+                if (depth === 0)
+                    return undefined;
+                depth--;
+                break;
+            case ",":
+                flush();
+                afterBar = false;
+                afterEquals = false;
+                break;
+            case "|":
+                if (!pending)
+                    return undefined;
+                afterBar = true;
+                break;
+            case "...":
+                // `input_query...` repeats the argument it follows; `k2, ...` repeats the last one.
+                flush();
+                if (args.length === 0)
+                    return undefined;
+                args[args.length - 1].repeated = true;
+                break;
+            case "=":
+                // `key = value, ...` is a settings tail (BigQuery); `tokenizer = x` names a parameter (text index).
+                if (!pending || pending.names.length !== 1)
+                    return undefined;
+                pending.named = true;
+                afterEquals = true;
+                break;
+            case "name":
+            case "quoted": {
+                if (pending && afterEquals) {
+                    // The value placeholder after `=` is an example, not a second argument.
+                    afterEquals = false;
+                    break;
+                }
+                if (pending && afterBar) {
+                    pending.names.push(tok.v);
+                    afterBar = false;
+                    break;
+                }
+                if (pending)
+                    return undefined;
+                pending = { names: [tok.v], quoted: tok.t === "quoted", optional: depth > 0, named: false };
+                break;
+            }
+        }
+    }
+    flush();
+    if (depth !== 0)
+        return undefined;
+    return { name, parenthesised: true, args };
+}