@mason-data/cli 0.0.0-stage → 0.7.0

package/dist/src/duck/explain.js

@@ -0,0 +1,238 @@
+/** a node of DuckDB's JSON, either form: `name` (EXPLAIN) or `type` with `intermediate_rows` and `timing` (the profile).
+ *  The profile's RESULT_COLLECTOR and EXECUTE are the result's plumbing, not the plan's: their child stands for them */
+export function planNodes(json) {
+    const list = Array.isArray(json) ? json : [json];
+    return list.flatMap((n) => {
+        const b = n;
+        const type = String(b.name ?? b.type ?? "");
+        const children = planNodes((b.children ?? []));
+        if (type === "RESULT_COLLECTOR" || type === "EXECUTE" || type === "EXPLAIN_ANALYZE")
+            return children;
+        const extra = { ...(b.extra_info ?? {}) };
+        const est = extra["Estimated Cardinality"];
+        delete extra["Estimated Cardinality"];
+        const node = { type: type === "SEQ_SCAN" ? "TABLE_SCAN" : type, extra, estimate: typeof est === "string" && /^\d+$/.test(est) ? Number(est) : null, children };
+        if (typeof b.intermediate_rows === "number") {
+            node.actualRows = b.intermediate_rows;
+            node.threadMs = Number(b.timing ?? 0) * 1000 + children.reduce((s, c) => s + (c.threadMs ?? 0), 0);
+        }
+        return [node];
+    });
+}
+/** a profile DuckDB's profiler wrote (`enable_profiling = 'json'`): the statement it timed, as it names it (`query.sql`,
+ *  the text it was given), and its plan. The name is what holds a profile to its statement (src/duck/store.ts
+ *  `explained`): the file is the last profiled statement's on the connection, whichever that was */
+export function profiled(doc) {
+    const d = doc;
+    return { sql: typeof d.query?.sql === "string" ? d.query.sql : null, plan: planNodes(d.operator ?? []) };
+}
+/** what EXPLAIN ANALYZE says of a statement DuckDB's profiler timed no operator of */
+export const UNPROFILED = "DuckDB's profiler recorded no operator of this statement: DuckDB answered it while planning it (as it answers a count(*) of a whole table from the table's row count), so the plan is its EXPLAIN, with the rows it answered";
+/** DuckDB's operator as DuckDB's own plan prints it (`HASH_GROUP_BY` is `Hash Group By`), a scan `Table Scan` as its
+ *  profiler names it: DuckDB's EXPLAIN says SEQ_SCAN for the same operator, and Postgres's `Seq Scan` says otherwise */
+export const operatorName = (type) => type.toLowerCase().split("_").map((w) => w === "n" ? "N" : w.charAt(0).toUpperCase() + w.slice(1)).join(" ");
+/** a name as Postgres writes one in a statement: bare when it is lower case letters, digits and `_`, else quoted */
+const quotedName = (s) => /^[a-z_][a-z0-9_$]*$/.test(s) ? s : `"${s.replaceAll('"', '""')}"`;
+/**
+ * A TABLE AS POSTGRES WRITES IT, each part quoted where it must be (`public.orders`, `public."Clients"`,
+ * `public."x; y"`): the ONE way EXPLAIN names a copy (its line, FORMAT JSON's `Mason.Copies[].Table`), which a statement
+ * can read back as it is (`mason mcp`'s tables and describe name tables by it too).
+ */
+export const qualifiedName = (t) => `${quotedName(t.schema)}.${quotedName(t.name)}`;
+/** DuckDB's `<file>.<schema>.<table>`, each part quoted where DuckDB quotes it (`memory.main."x; drop.t"`), as its last
+ *  two parts: a `.` inside a quoted part is the name's */
+export function duckTableName(text) {
+    const parts = [];
+    let at = 0;
+    while (at <= text.length) {
+        if (text[at] === '"') {
+            let out = "", end = at + 1;
+            for (; end < text.length; end++) {
+                if (text[end] === '"') {
+                    if (text[end + 1] === '"') {
+                        out += '"';
+                        end++;
+                        continue;
+                    }
+                    break;
+                }
+                out += text[end];
+            }
+            parts.push(out);
+            at = end + 2; // past the closing quote and the dot
+        }
+        else {
+            const dot = text.indexOf(".", at);
+            parts.push(text.slice(at, dot < 0 ? undefined : dot));
+            at = dot < 0 ? text.length + 1 : dot + 1;
+        }
+    }
+    return parts.length >= 2 ? { schema: parts[parts.length - 2], name: parts[parts.length - 1] } : null;
+}
+/** the copy a scan reads (DuckDB's `<file>.<schema>.<table>`) */
+export const scannedTable = (n) => {
+    const t = n.extra.Table;
+    return typeof t === "string" ? duckTableName(t) : null;
+};
+export function scannedTables(nodes) {
+    const out = [];
+    const walk = (n) => {
+        const t = scannedTable(n);
+        if (t !== null && !out.some((o) => o.schema === t.schema && o.name === t.name))
+            out.push(t);
+        n.children.forEach(walk);
+    };
+    nodes.forEach(walk);
+    return out;
+}
+/** the sync's states as EXPLAIN takes them across to the executor's thread (src/api/postgres/server.ts `duckExplain`):
+ *  its facts alone, read at the EXPLAIN */
+export const copyFacts = (states) => states.map((s) => ({ source: s.source, phase: s.phase, error: s.error, ...(s.rowsSoFar !== undefined ? { rowsSoFar: s.rowsSoFar } : {}), ...(s.kept !== undefined ? { kept: s.kept } : {}) }));
+/** an ISO instant as EXPLAIN says one: `2026-10-27 19:58:39 UTC` */
+const utc = (iso) => `${iso.slice(0, 19).replace("T", " ")} UTC`;
+const UNKNOWN = "how it is kept current is unknown: the sync has no record of it";
+/**
+ * HOW A COPY IS KEPT CURRENT, in EXPLAIN's words, from the sync's one derivation (src/duck/sync.ts `KeptCurrent`, which
+ * `mason status` renders too): a copy not yet copied or whose copy failed says so first; then polled with its last
+ * poll, served as copied and skipped by the passes (to be copied again, caught up first, over the poll's row limit),
+ * copied again whole, or, in replication mode, its part in the stream, how far the stream has applied and why it is
+ * not running. "kept current" is said only of a copy that is (`keptCurrentBy`).
+ */
+export function keptText(f) {
+    if (f === null)
+        return UNKNOWN;
+    if (f.phase === "waiting")
+        return "waiting for its copy to start";
+    if (f.phase === "copying")
+        return `being copied (${count(f.rowsSoFar ?? 0)} rows so far): kept current once copied`;
+    if (f.phase === "failed")
+        return `its last copy failed: ${(f.error ?? "mason logs says why").split("\n")[0]}`;
+    const k = f.kept;
+    if (k === undefined)
+        return UNKNOWN;
+    if (k.by === "poll") {
+        const asOf = k.polledAt === null ? "no poll recorded yet" : `as of ${utc(k.polledAt)}`;
+        if (k.pending !== undefined)
+            return `NOT kept current (${asOf}): ${k.pending}`;
+        if (k.unsynced !== undefined)
+            return `NOT kept current (${k.unsynced}; ${asOf}): mason sync ${f.source} polls it now`;
+        return `kept current by polling, ${k.polledAt === null ? "no poll recorded yet" : `last poll ${utc(k.polledAt)}`}`;
+    }
+    if (k.by === "recopy")
+        return `kept current by copying it again whole, ${k.copiedAt === null ? "no copy recorded yet" : `last copied ${utc(k.copiedAt)}`}`;
+    const stream = `logical replication (slot ${k.slot})`;
+    const where = `applied through ${k.position}, ${k.lastCommit === null ? "no change applied this run" : `last change committed ${utc(k.lastCommit)}`}`;
+    if (k.state === "streamed")
+        return k.notRunning === null ? `kept current by ${stream}, ${where}`
+            : `streamed by ${stream}, ${where}; NOT kept current now, the stream is not running: ${k.notRunning}`;
+    return k.state === "left-out" ? `NOT kept current: left out of ${stream}: ${k.why ?? ""}`
+        : `${stream}, not streamed yet: ${k.why ?? ""}`;
+}
+/** what keeps a copy current NOW, as FORMAT JSON's `Kept Current By` says it: null for a copy nothing does (not copied
+ *  yet, skipped by the passes, over the poll's row limit, not streamed, the stream down), so a reader keying on it is
+ *  never told of a mode that is not at work on it */
+export function keptCurrentBy(f) {
+    const k = f?.phase === "copied" ? f.kept : undefined;
+    if (k === undefined)
+        return null;
+    if (k.by === "poll")
+        return k.pending === undefined && k.unsynced === undefined ? "polling" : null;
+    if (k.by === "recopy")
+        return "copying again";
+    return k.state === "streamed" && k.notRunning === null ? "logical replication" : null;
+}
+/** how a copy is kept current, as FORMAT JSON's keys say it (`keptText`'s facts) */
+function keptJson(f) {
+    const k = f?.phase === "copied" ? f.kept : undefined;
+    return {
+        "Kept Current By": keptCurrentBy(f), "Kept Current": keptText(f),
+        ...(k?.by === "poll" ? { "Last Poll": k.polledAt } : {}),
+        ...(k?.by === "replication" ? { Slot: k.slot, "Stream State": k.state, "Applied Through": k.position, "Last Change Committed": k.lastCommit, "Stream Not Running": k.notRunning } : {}),
+    };
+}
+const ms = (x) => x.toFixed(3);
+const count = (n) => n.toLocaleString("en-US");
+const detail = (v) => Array.isArray(v) ? v.join(", ") : v;
+/** the details a plan line carries without VERBOSE: everything but the projections and filters, which VERBOSE adds */
+const VERBOSE_ONLY = new Set(["Projections", "Filters", "Table", "Type"]);
+function copyLine(c) {
+    const rows = c.rows === null ? "rows unknown" : `about ${count(c.rows)} rows`;
+    return `${c.table} (${rows}; ${keptText(c.facts)})`;
+}
+/** EXPLAIN as Postgres's text format: one line a row */
+export function explainText(e, o) {
+    const lines = [];
+    lines.push(e.copies.length === 0 ? "Mason: answered by DuckDB, reading none of its copies"
+        : e.copies.length === 1 ? `Mason: answered by DuckDB from its copy of ${copyLine(e.copies[0])}` : "Mason: answered by DuckDB from its copies of");
+    if (e.copies.length > 1)
+        for (const c of e.copies)
+            lines.push(`  ${copyLine(c)}`);
+    lines.push(`  Sent to DuckDB: ${e.sql}`);
+    lines.push(`  Computed by Mason after DuckDB: ${e.computed.length === 0 ? "nothing" : e.computed.join("; ")}`);
+    if (e.functions.length > 0)
+        lines.push(`  Mason's functions DuckDB calls: ${e.functions.join(", ")}`);
+    for (const name of o.ignored)
+        lines.push(`  Note: ${name} has no DuckDB meaning: ignored`);
+    if (e.analyzed?.profiled === false)
+        lines.push(`  Note: ${UNPROFILED}`);
+    else if (e.analyzed !== undefined && o.timing)
+        lines.push("  Note: an operator's thread time is DuckDB's, summed over the threads that ran it and the operators under it: a parallel one's exceeds the Execution Time");
+    const walk = (n, depth) => {
+        const lead = depth === 0 ? "" : `${" ".repeat(6 * depth - 4)}->  `;
+        const on = scannedTable(n);
+        const est = o.costs && n.estimate !== null ? `  (rows=${n.estimate})` : "";
+        const actual = n.actualRows === undefined ? "" : ` (actual rows=${n.actualRows}${o.timing && n.threadMs !== undefined ? `, thread time=${ms(n.threadMs)} ms` : ""})`;
+        lines.push(`${lead}${operatorName(n.type)}${on === null ? "" : ` on ${qualifiedName(on)}`}${est}${actual}`);
+        const pad = " ".repeat(depth === 0 ? 2 : 6 * depth + 2);
+        for (const [k, v] of Object.entries(n.extra))
+            if (o.verbose ? k !== "Table" : !VERBOSE_ONLY.has(k))
+                lines.push(`${pad}${k}: ${detail(v)}`);
+        for (const c of n.children)
+            walk(c, depth + 1);
+    };
+    e.plan.forEach((n) => walk(n, 0));
+    if (e.analyzed !== undefined && o.summary) {
+        const p = e.analyzed.planning, x = e.analyzed.execution;
+        lines.push(`Planning Time: ${ms(p.analysisMs + p.rewriteMs + p.duckdbMs)} ms (Postgres's analysis ${ms(p.analysisMs)}, Mason's rewrite ${ms(p.rewriteMs)}, DuckDB's planning ${ms(p.duckdbMs)})`);
+        lines.push(`Execution Time: ${ms(x.duckdbMs + x.masonMs)} ms (DuckDB ${ms(x.duckdbMs)}, Mason's conversion and exact numerics ${ms(x.masonMs)})`);
+    }
+    return lines;
+}
+/** EXPLAIN as Postgres's JSON format: `[{ "Plan": { "Node Type", "Plans", "Plan Rows", "Actual Rows" … } }]` where the
+ *  meaning is Postgres's, DuckDB's operator names, and what is DuckDB's own in keys of their own — its thread time is
+ *  `DuckDB Thread Time`, never `Actual Total Time`, which a plan viewer draws as elapsed */
+export function explainJson(e, o) {
+    const node = (n) => {
+        const on = scannedTable(n);
+        return {
+            "Node Type": operatorName(n.type),
+            ...(on === null ? {} : { "Relation Name": on.name, Schema: on.schema }),
+            ...(o.costs && n.estimate !== null ? { "Plan Rows": n.estimate } : {}),
+            ...(n.actualRows === undefined ? {} : { "Actual Rows": n.actualRows, "Actual Loops": 1, ...(o.timing && n.threadMs !== undefined ? { "DuckDB Thread Time": Number(ms(n.threadMs)) } : {}) }),
+            ...Object.fromEntries(Object.entries(n.extra).filter(([k]) => o.verbose ? k !== "Table" : !VERBOSE_ONLY.has(k)).map(([k, v]) => [`DuckDB ${k}`, v])),
+            ...(n.children.length === 0 ? {} : { Plans: n.children.map(node) }),
+        };
+    };
+    // Postgres's plan has one root; DuckDB's has one too (a statement's), kept as a list should it ever have more
+    const roots = e.plan.map(node);
+    const top = {
+        Plan: roots.length === 1 ? roots[0] : { "Node Type": "DuckDB Plans", Plans: roots },
+        Mason: {
+            "Answered By": "DuckDB",
+            Copies: e.copies.map((c) => ({ Table: c.table, "Estimated Rows": c.rows, ...keptJson(c.facts) })),
+            "Sent To DuckDB": e.sql,
+            "Computed After DuckDB": e.computed,
+            "Mason Functions DuckDB Calls": e.functions,
+            ...(o.ignored.length === 0 ? {} : { Ignored: o.ignored.map((n) => `${n} has no DuckDB meaning`) }),
+            ...(e.analyzed?.profiled === false ? { Note: UNPROFILED } : {}),
+        },
+    };
+    if (e.analyzed !== undefined && o.summary) {
+        const p = e.analyzed.planning, x = e.analyzed.execution;
+        top["Planning Time"] = Number(ms(p.analysisMs + p.rewriteMs + p.duckdbMs));
+        top["Execution Time"] = Number(ms(x.duckdbMs + x.masonMs));
+        top.Mason["Planning"] = { "Postgres Analysis": Number(ms(p.analysisMs)), "Mason Rewrite": Number(ms(p.rewriteMs)), "DuckDB Planning": Number(ms(p.duckdbMs)) };
+        top.Mason["Execution"] = { DuckDB: Number(ms(x.duckdbMs)), "Mason Conversion": Number(ms(x.masonMs)) };
+    }
+    return JSON.stringify([top], null, 2);
+}