@mason-data/cli 0.0.0-stage → 0.6.2

package/dist/src/api/postgres/explain.js

@@ -0,0 +1,143 @@
+import { parseSql, PostgresSyntaxError } from "./parser.js";
+import { Unsupported } from "./translate.js";
+const bodyOf = (n) => Object.values(n)[0];
+/** Postgres's defGetBoolean: no value is true; true / false / on / off / 1 / 0 */
+function booleanOf(name, arg, at) {
+    if (arg === undefined)
+        return true;
+    const kind = Object.keys(arg)[0];
+    if (kind === "Integer") {
+        const v = Number(bodyOf(arg).ival ?? 0);
+        if (v === 0 || v === 1)
+            return v === 1;
+    }
+    if (kind === "String" || kind === "Boolean") {
+        const v = kind === "Boolean" ? String(bodyOf(arg).boolval ?? false) : String(bodyOf(arg).sval ?? "").toLowerCase();
+        if (["true", "on"].includes(v))
+            return true;
+        if (["false", "off"].includes(v))
+            return false;
+    }
+    throw new Unsupported(`${name} requires a Boolean value`, at, "42601"); // defGetBoolean: a syntax error
+}
+/** Postgres's defGetString: an option's value as its text (an integer's digits, a float's as written, a Boolean's
+ *  true / false); no value at all is its 42601. Compared case-sensitively after, as explain.c does (`'JSON'` is not json). */
+function stringOf(name, arg, at) {
+    if (arg === undefined)
+        throw new Unsupported(`${name} requires a parameter`, at, "42601");
+    const kind = Object.keys(arg)[0];
+    const b = bodyOf(arg);
+    if (kind === "Integer")
+        return String(b.ival ?? 0); // a 0 is no field at all in the parse tree
+    if (kind === "Float")
+        return String(b.fval ?? "");
+    if (kind === "Boolean")
+        return String(b.boolval ?? false);
+    return String(b.sval ?? "");
+}
+/** an EXPLAIN's options as Postgres reads them (explain.c ParseExplainOptionList), its errors in its words */
+export function explainOptions(options) {
+    const o = { analyze: false, verbose: false, costs: true, timing: true, summary: false, format: "text", ignored: [] };
+    let timingSet = false, summarySet = false, wal = false, serialize = false;
+    for (const d of options) {
+        const b = bodyOf(d);
+        const name = String(b.defname ?? "");
+        const at = typeof b.location === "number" ? b.location : null;
+        switch (name) {
+            case "analyze":
+                o.analyze = booleanOf("analyze", b.arg, at);
+                break;
+            case "verbose":
+                o.verbose = booleanOf("verbose", b.arg, at);
+                break;
+            case "costs":
+                o.costs = booleanOf("costs", b.arg, at);
+                break;
+            case "timing":
+                o.timing = booleanOf("timing", b.arg, at);
+                timingSet = true;
+                break;
+            case "summary":
+                o.summary = booleanOf("summary", b.arg, at);
+                summarySet = true;
+                break;
+            case "buffers":
+            case "settings":
+            case "memory":
+                if (booleanOf(name, b.arg, at))
+                    o.ignored.push(name.toUpperCase());
+                break;
+            case "wal":
+                wal = booleanOf(name, b.arg, at);
+                if (wal)
+                    o.ignored.push("WAL");
+                break;
+            case "serialize": {
+                // SERIALIZE alone is TEXT; else OFF, NONE, TEXT or BINARY, and nothing else (no Boolean: Postgres 17 refuses `on`)
+                const v = b.arg === undefined ? "text" : stringOf(name, b.arg, at);
+                if (!["none", "off", "text", "binary"].includes(v))
+                    throw new Unsupported(`unrecognized value for EXPLAIN option "serialize": "${v}"`, at, "22023");
+                serialize = v === "text" || v === "binary";
+                if (serialize)
+                    o.ignored.push("SERIALIZE");
+                break;
+            }
+            case "generic_plan":
+                if (booleanOf(name, b.arg, at))
+                    throw new Unsupported(`EXPLAIN (GENERIC_PLAN) is not supported by Mason (Mason gap explain-option)`, at, "0A000", "bind the parameters' values and EXPLAIN that: DuckDB plans the statement with them");
+                break;
+            case "format": {
+                const v = stringOf(name, b.arg, at);
+                if (v === "text" || v === "json") {
+                    o.format = v;
+                    break;
+                }
+                if (v === "xml" || v === "yaml")
+                    throw new Unsupported(`EXPLAIN (FORMAT ${v.toUpperCase()}) is not supported by Mason (Mason gap explain-option)`, at, "0A000", "write FORMAT TEXT or FORMAT JSON");
+                throw new Unsupported(`unrecognized value for EXPLAIN option "format": "${v}"`, at, "22023");
+            }
+            default: throw new Unsupported(`unrecognized EXPLAIN option "${name}"`, at, "42601");
+        }
+    }
+    if (wal && !o.analyze)
+        throw new Unsupported("EXPLAIN option WAL requires ANALYZE", null, "22023");
+    if (timingSet && o.timing && !o.analyze)
+        throw new Unsupported("EXPLAIN option TIMING requires ANALYZE", null, "22023");
+    if (serialize && !o.analyze)
+        throw new Unsupported("EXPLAIN option SERIALIZE requires ANALYZE", null, "22023");
+    if (!summarySet)
+        o.summary = o.analyze;
+    if (!o.analyze)
+        o.timing = false;
+    return o;
+}
+/** a tree without its offsets into the text: every field @pgsql/types declares as one (an IN list's bounds too) */
+const OFFSETS = new Set(["location", "name_location", "stmt_location", "stmt_len", "list_start", "list_end", "rexpr_list_start", "rexpr_list_end"]);
+const stripped = (n) => JSON.stringify(n, (k, v) => OFFSETS.has(k) ? undefined : v);
+/**
+ * The statement an EXPLAIN explains, as its own text: the slice of the EXPLAIN's text that parses, alone, to the tree
+ * the EXPLAIN holds (its first start that does, tried at each `SELECT`, `WITH`, `VALUES`, `TABLE` or `(` after the
+ * keyword) — so it is analysed, planned and refused as the statement itself is. Null where no slice does (a Mason gap:
+ * the caller refuses it).
+ */
+export async function explainedText(text, query) {
+    const want = stripped(query);
+    const start = /^\s*explain\b/i.exec(text.replace(/\/\*[\s\S]*?\*\//g, (c) => " ".repeat(c.length)));
+    const from = start === null ? 0 : start[0].length;
+    for (const m of text.slice(from).matchAll(/\b(select|with|values|table)\b|\(/gi)) {
+        const at = from + m.index;
+        let parsed;
+        try {
+            parsed = await parseSql(text.slice(at));
+        }
+        catch (e) {
+            if (e instanceof PostgresSyntaxError)
+                continue;
+            throw e;
+        } // a slice that is no statement: the next start
+        const stmts = parsed.stmts ?? [];
+        if (stmts.length === 1 && stmts[0]?.stmt !== undefined && stripped(stmts[0].stmt) === want)
+            return { text: text.slice(at), offset: at };
+    }
+    return null;
+}

package/dist/src/api/postgres/listener.js

@@ -0,0 +1,60 @@
+/**
+ * WHERE THE POSTGRES API LISTENS, READ IN ONE PLACE: `MASON_PG_PORT` (default 5433) and `MASON_PG_HOST`
+ * (default: the host its caller names, 127.0.0.1 for a run). `mason start` reads it to listen and to record where its run's listener is for
+ * `mason user` and `mason status` (src/cli/dev.ts); in the ClickHouse stack this was cut from, its Query API's
+ * listener read it too, and its service's config check, to refuse a bad port at the start instead of a child
+ * crash-looping — one reader, one default, so what the CLI prints is where the API listens (CLAUDE.md "Fix the
+ * class": a fact derived in several places is one module's).
+ *
+ * WHETHER IT IS TLS, READ HERE TOO (the DuckDB run once fixed its host to 127.0.0.1 and its TLS to none, and on
+ * the test server a TLS proxy had to be written beside Mason for a hosted BI tool): `MASON_PG_TLS_CERT` / `MASON_PG_TLS_KEY`
+ * (PEM files, both or neither) and the plain-listener rule (src/api/postgres/plain.ts) are `pgListenerSettings`'s, read
+ * by `mason start` and `mason serve` alike (src/cli/pg-tls.ts `runListenerSettings`) — `mason start --self-signed-tls` names its own files through
+ * the same two variables (src/cli/pg-tls.ts), never a second reader.
+ */
+import * as fs from "node:fs";
+import { parsePort } from "../../lib/port.js";
+import { plainListener } from "./plain.js";
+export const DEFAULT_PG_PORT = 5433;
+/** The listener's address from `env`, or the problem with its port (`parsePort`'s words). */
+export function pgListenerAddress(env, apiHost) {
+    const port = parsePort("MASON_PG_PORT", env.MASON_PG_PORT, DEFAULT_PG_PORT);
+    if (port.problem !== undefined)
+        return { problem: port.problem };
+    return { host: env.MASON_PG_HOST || apiHost, port: port.port };
+}
+/** Where and how the listener listens, from `env` (`MASON_PG_HOST`, `MASON_PG_PORT`, `MASON_PG_TLS_CERT`,
+ *  `MASON_PG_TLS_KEY`, `MASON_PG_PLAIN`), or why it cannot: a bad port, half a certificate, plain off loopback. Reads no
+ *  file, so a start can be refused before anything is made. */
+export function pgListenerSettings(env, apiHost) {
+    const address = pgListenerAddress(env, apiHost);
+    if ("problem" in address)
+        return { problem: address.problem, kind: "port" };
+    const cert = env.MASON_PG_TLS_CERT || undefined, key = env.MASON_PG_TLS_KEY || undefined;
+    if ((cert && !key) || (key && !cert))
+        return { problem: "MASON_PG_TLS_CERT and MASON_PG_TLS_KEY go together (PEM files): set both, or neither for a plain listener", kind: "tls_pair" };
+    const plain = plainListener(address.host, env.MASON_PG_PLAIN, Boolean(cert && key));
+    if (plain !== null)
+        return plain;
+    return { ...address, tls: cert && key ? { cert, key } : null };
+}
+/** The certificate and key the settings name, read; or why not. */
+export function readPgTls(files) {
+    try {
+        return { cert: fs.readFileSync(files.cert), key: fs.readFileSync(files.key) };
+    }
+    catch (err) {
+        return { problem: `cannot read MASON_PG_TLS_CERT / MASON_PG_TLS_KEY: ${err instanceof Error ? err.message : String(err)}`, kind: "tls_read" };
+    }
+}
+/**
+ * The Postgres API's address as a URL without credentials — `postgresql://<host>:<port>/<database>` — the form
+ * `mason start` records in the project's config and `mason status` prints; `mason user add` puts a user and its
+ * password in it (src/cli/pg-url.ts `connectionUrl`). A host of every interface (0.0.0.0, ::) is written as
+ * localhost: that is where a tool on the machine reaches it. A TLS listener's ends `?sslmode=require`, the libpq word every driver takes:
+ * it refuses a plain connection, so the URL a tool is handed says so.
+ */
+export function postgresApiUrl(address, database, o = {}) {
+    const host = address.host === "0.0.0.0" || address.host === "::" ? "localhost" : address.host;
+    return `postgresql://${host.includes(":") ? `[${host}]` : host}:${address.port}/${encodeURIComponent(database)}${o.tls ? "?sslmode=require" : ""}`;
+}

package/dist/src/api/postgres/parser-worker.js

@@ -0,0 +1,58 @@
+/**
+ * THE POSTGRES PARSER, in a worker thread (src/api/postgres/parser.ts asks it, and says why it is recycled).
+ *
+ * `libpg-query` is Postgres 18's own parser compiled to WebAssembly: the same grammar a statement would meet on a real
+ * Postgres, so "does this parse" is never a question this repo answers for itself.
+ *
+ * Protocol: a request is the SQL text; the reply is `{ ok: true, result }` (the parse tree) or
+ * `{ ok: false, message, cursorPosition }` (a syntax error, with the 0-based offset: libpg-query
+ * has already converted Postgres's 1-based cursor), or `{ ok: false, fatal: true, message }` when
+ * the parser itself could not start. After every reply the shared signal is set to 1 and notified.
+ */
+import { setFlagsFromString } from "node:v8";
+import { workerData } from "node:worker_threads";
+const { port, signal } = workerData;
+const answered = () => { Atomics.store(signal, 0, 1); Atomics.notify(signal, 0); };
+// NEVER OPTIMIZE THE PARSER'S WEBASSEMBLY. By default V8 re-compiles a hot WebAssembly function with its
+// optimizing compiler in the background, and for this module that costs ~900 MB of resident memory, kept for
+// the life of the process (the test server, node 24.21 on Linux: emit peaks at 995 MB with it and 107 MB
+// without, `/sys/fs/cgroup/memory.peak` of the bootstrap container, which is limited to 1 GiB and killed it).
+// A statement is short: the baseline code is fast enough.
+//
+// WHAT TURNS IT OFF IS MEASURED, NOT INFERRED, because the flags read at runtime differ from the ones read at
+// start-up and differ between platforms. `--liftoff-only` on the command line works (93 MB) but is read once,
+// when the engine starts, and set here does nothing (995 MB). `--wasm-tiering-budget=<max>` set here worked on
+// macOS (the first fix, 117 MB) and does NOTHING on the test server's Linux build (997 MB as a CLI flag,
+// 995 MB from here) — which is how a reload that had fitted stopped fitting once a few more parses were added.
+// Each of `--no-wasm-dynamic-tiering` and `--no-wasm-tier-up` alone is still 995 MB; BOTH, set here before
+// `libpg-query` compiles its module below, are 107 MB on the test server (three runs) — the two switches that
+// `--liftoff-only` implies. Re-measure on the test server before changing this line: a wrong flag is silent.
+setFlagsFromString("--no-wasm-dynamic-tiering --no-wasm-tier-up");
+// THE PARSER'S OWN START-UP FAILURE IS THE REPLY. The caller is blocked in `Atomics.wait` and can observe nothing
+// but the port and the signal, so a `libpg-query` that is not installed (no `npm ci`) or a WebAssembly that will not
+// load is reported as a fatal reply carrying the real message — not as a 20 s silence guessed at from outside.
+let parser;
+try {
+    parser = await import("libpg-query");
+    await parser.loadModule();
+}
+catch (e) {
+    const fatal = { ok: false, fatal: true, message: `the Postgres parser could not start: ${e.message}` };
+    port.on("message", () => { port.postMessage(fatal); answered(); });
+    port.postMessage(fatal);
+    answered();
+    throw e;
+}
+port.on("message", (sql) => {
+    try {
+        const result = parser.parseSync(String(sql));
+        port.postMessage({ ok: true, result });
+    }
+    catch (e) {
+        const err = e instanceof parser.SqlError ? e : null;
+        port.postMessage({ ok: false, message: e.message, cursorPosition: err?.sqlDetails?.cursorPosition ?? null });
+    }
+    finally {
+        answered();
+    }
+});

package/dist/src/api/postgres/parser.js

@@ -0,0 +1,121 @@
+/**
+ * THE POSTGRES PARSER FOR THE POSTGRES LISTENER: Postgres 18's own grammar (`libpg-query`, WebAssembly), in a worker, asked
+ * asynchronously. What a tool sends either parses here exactly as it would on a real Postgres, or fails with
+ * Postgres's own message at Postgres's own position (the Postgres API's first invariant, from the day it was planned).
+ *
+ * WHY A WORKER, AND WHY IT IS RECYCLED. The parser leaks ~110 KB of its WebAssembly memory on EVERY call (measured
+ * on 18.1.5, the version installed), and a listener serving a BI tool parses an unbounded number of
+ * distinct statements — so the worker is replaced after `RECYCLE_AFTER` parses, which bounds the leak to a few tens of
+ * megabytes. And V8 would re-compile the parser's WebAssembly with its optimising compiler for ~900 MB of resident
+ * memory, which parser-worker.ts turns off inside the worker: it sets the flags, loads the module, and
+ * answers each text on the port with the tree or the syntax error and its offset.
+ *
+ * NOTHING HERE IS IMPORTED AT STARTUP: the worker is started on the first statement, and a parser that cannot start
+ * fails THAT connection's statements with the reason (`libpg-query` is not installed), never the listener's start.
+ */
+import { MessageChannel, Worker } from "node:worker_threads";
+import { fileURLToPath } from "node:url";
+import { runnable } from "../../lib/runnable.js";
+/** A Postgres syntax error: Postgres's own message, with the 0-based offset into the text. */
+export class PostgresSyntaxError extends Error {
+    offset;
+    constructor(message, offset) { super(message); this.offset = offset; }
+}
+/** parses per worker before it is replaced (~110 KB leaked each: ~45 MB at most per worker's life) */
+export const RECYCLE_AFTER = 400;
+/** one parse's deadline, the worker's start (WASM instantiation, ~100 ms) included: past it the worker is dead */
+const TIMEOUT_MS = 20_000;
+class ParserWorker {
+    worker;
+    port;
+    /** replies arrive in the order the texts were sent: one queue */
+    queue = [];
+    sent = 0;
+    dead = null;
+    constructor() {
+        const { port1, port2 } = new MessageChannel();
+        const signal = new Int32Array(new SharedArrayBuffer(4));
+        const file = new URL("./parser-worker.ts", import.meta.url);
+        // `execArgv: []`: the worker must not inherit this process's Node flags — a process run as `node --input-type=module
+        // -e …` would hand it `--input-type`, which Node refuses for a file, and the worker would die before answering
+        this.worker = new Worker(runnable(fileURLToPath(file)), { workerData: { port: port2, signal }, transferList: [port2], execArgv: [], argv: [] });
+        this.worker.on("error", (e) => this.die(new Error(`the Postgres parser's worker failed: ${e.message}`)));
+        this.worker.on("exit", (code) => this.die(new Error(`the Postgres parser's worker exited with code ${code}`)));
+        this.worker.unref();
+        this.port = port1;
+        this.port.unref();
+        this.port.on("message", (reply) => {
+            if (!reply.ok && "fatal" in reply) {
+                this.die(new Error(reply.message));
+                return;
+            }
+            const p = this.queue.shift();
+            if (p === undefined)
+                return; // the worker's own fatal reply sent before any request: handled above
+            clearTimeout(p.timer);
+            if (reply.ok)
+                p.resolve(reply.result);
+            else
+                p.reject(new PostgresSyntaxError(reply.message, reply.cursorPosition));
+        });
+    }
+    /** everything pending fails with the reason, and so does everything after */
+    die(e) {
+        if (this.dead !== null)
+            return;
+        this.dead = e;
+        for (const p of this.queue.splice(0)) {
+            clearTimeout(p.timer);
+            p.reject(e);
+        }
+    }
+    get alive() { return this.dead === null; }
+    parse(sql) {
+        if (this.dead !== null)
+            return Promise.reject(this.dead);
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                this.die(new Error(`the Postgres parser did not answer within ${TIMEOUT_MS / 1000} s — its worker (src/api/postgres/parser-worker.ts) is dead or hung; its own error, if it printed one, is on stderr`));
+                void this.worker.terminate();
+            }, TIMEOUT_MS);
+            timer.unref();
+            this.queue.push({ resolve, reject, timer });
+            this.sent++;
+            this.port.postMessage(sql);
+        });
+    }
+    async stop() {
+        this.die(new Error("the Postgres parser was stopped"));
+        await this.worker.terminate();
+    }
+}
+let current = null;
+/** workers past their recycle count, kept until their last reply is in */
+const retiring = new Set();
+/** The parse tree of one text (libpg-query's `ParseResult`, Postgres 18's grammar), or a PostgresSyntaxError. */
+export async function parseSql(sql) {
+    // an empty string is no statement to Postgres (EmptyQueryResponse; pgjdbc's isValid() sends one), where libpg_query
+    // throws "Query cannot be empty" — whitespace, a `;` or a comment alone already parse to no statement
+    if (sql === "")
+        return { version: 0, stmts: [] };
+    if (current === null || !current.alive || current.sent >= RECYCLE_AFTER) {
+        if (current !== null && current.alive) {
+            // let its queued replies land, then terminate it
+            const old = current;
+            retiring.add(old);
+            void old.parse("SELECT 1").then(() => undefined, () => undefined).finally(() => { retiring.delete(old); void old.stop(); });
+        }
+        current = new ParserWorker();
+    }
+    return current.parse(sql);
+}
+/** Stops the parser (tests, and the API's shutdown). */
+export async function stopParser() {
+    const w = current;
+    current = null;
+    if (w !== null)
+        await w.stop();
+    for (const r of retiring)
+        await r.stop();
+    retiring.clear();
+}

package/dist/src/api/postgres/plain.js

@@ -0,0 +1,33 @@
+/**
+ * A PLAIN LISTENER OFF LOOPBACK IS A DECISION, NOT A DEFAULT: the token travels as the password in
+ * clear, so anyone on the network path reads it and queries as that token until it expires. On loopback nothing
+ * travels; elsewhere the operator says which it is — TLS (`MASON_PG_TLS_CERT` / `MASON_PG_TLS_KEY`), or
+ * `MASON_PG_PLAIN=1` for a port published on a private network (a compose or Kubernetes network, a loopback publish
+ * on the host). Read by the API's listener and by the service's config check alike, so the refusal is said at the
+ * start, never as a child crash-looping. docs/POSTGRES-API.md "Connecting".
+ */
+/** a host on which a plain listener crosses no network */
+export function isLoopback(host) {
+    return host === "localhost" || host === "::1" || /^127\.\d+\.\d+\.\d+$/.test(host);
+}
+/** A problem as one line: what, why, the way out — the service's refusal and log. */
+export function problemText(p) {
+    return `${p.problem}${p.why ? `: ${p.why}` : ""}${p.fix ? `. ${p.fix}` : ""}`;
+}
+/** The problem with listening plain on `host`, or null: TLS set, a loopback host, or `MASON_PG_PLAIN=1` is fine. */
+export function plainListener(host, plain, tls) {
+    if (tls || isLoopback(host))
+        return null;
+    if (plain === "1" || plain === "true")
+        return null;
+    if (plain !== undefined && plain !== "" && plain !== "0" && plain !== "false")
+        return { problem: `MASON_PG_PLAIN=${plain} is not a setting: 1 (a plain listener on a private network) or unset`, kind: "plain_value" };
+    return { problem: `the Postgres API would listen PLAIN on ${host}, off loopback`, why: "a token travels as the password in clear on that network", kind: "plain",
+        fix: "Set MASON_PG_TLS_CERT and MASON_PG_TLS_KEY (PEM files) to require TLS, or MASON_PG_PLAIN=1 to say the port is published on a private network " +
+            "(a compose or Kubernetes network, a 127.0.0.1 publish on the host) — docs/POSTGRES-API.md \"Connecting\"" };
+}
+/** `plainListener` as one line, as a check at the start prints it. */
+export function plainListenerProblem(host, plain, tls) {
+    const p = plainListener(host, plain, tls);
+    return p === null ? null : problemText(p);
+}