@mason-data/cli 0.8.0 → 0.10.0

package/dist/src/api/postgres/translate.js

@@ -55,6 +55,45 @@ function collect(n, kind, out = []) {
     }
     return out;
 }
+/**
+ * Whether a statement of no table has a construct that can order text, read off the raw tree: a sort (ORDER BY, a
+ * window's, an aggregate's or WITHIN GROUP's), <, >, <=, >= (BETWEEN, ANY / ALL and a subquery's among them),
+ * GREATEST / LEAST, min / max, upper / lower / initcap (which map case, issue 406), and a COLLATE (`'a' = 'A' COLLATE mason_ci`). One without (`SELECT 1`, `SELECT version()`, `current_setting(…)`, `'a' = 'A'`) is the
+ * catalog's without asking its analysis, which costs a statement several milliseconds (`ordersText`, issue 122): a
+ * statement it says may order is asked, so a false "may" costs time, never an answer.
+ */
+export function mayOrderText(n) {
+    if (Array.isArray(n))
+        return n.some(mayOrderText);
+    if (n === null || typeof n !== "object")
+        return false;
+    const named = (x) => collect(x ?? [], "String").map((s) => String(bodyOf(s).sval ?? ""));
+    for (const [k, v] of Object.entries(n)) {
+        if ((k === "sortClause" || k === "orderClause" || k === "agg_order") && Array.isArray(v) && v.length > 0)
+            return true;
+        if (k === "MinMaxExpr")
+            return true;
+        // a COLLATE names a collation the source declared, whose rules the catalog's copy of it does not hold: a
+        // nondeterministic one equates otherwise, and the synced data's plan refuses it by name (`ordersText` asks)
+        if (k === "CollateClause")
+            return true;
+        if ((k === "A_Expr" || k === "SubLink") && v !== null && typeof v === "object") {
+            const b = v;
+            if (typeof b.kind === "string" && b.kind.includes("BETWEEN"))
+                return true;
+            if (named(b.name ?? b.operName).some((o) => ORDERING_OPERATORS.has(o)))
+                return true;
+        }
+        // min / max order text; upper / lower / initcap map its case, which the catalog does by its own rules (issue 406)
+        if (k === "FuncCall" && v !== null && typeof v === "object" && COLLATED_CALLS.has((named(v.funcname).at(-1) ?? "").toLowerCase()))
+            return true;
+        if (mayOrderText(v))
+            return true;
+    }
+    return false;
+}
+const ORDERING_OPERATORS = new Set(["<", ">", "<=", ">="]);
+const COLLATED_CALLS = new Set(["min", "max", "upper", "lower", "initcap"]);
 const INFORMATION_SCHEMA = "information_schema";
 const PG_CATALOG = "pg_catalog";
 /** Postgres's IsTransactionExitStmt: what a failed block still accepts. `ROLLBACK TO` and `PREPARE TRANSACTION` are among
@@ -133,6 +172,8 @@ export function duckTranslate(stmt, text) {
     const ctes = (s.withClause?.ctes ?? []).map((n) => bodyOf(n));
     const names = new Set(ctes.map((c) => String(c.ctename ?? "")));
     const relations = collect(s, "RangeVar").map((n) => bodyOf(n)).filter((r) => r.schemaname !== undefined || !names.has(String(r.relname ?? "")));
+    if (relations.length === 0)
+        return mayOrderText(s) ? { kind: "catalog", fromless: { names: pgColumnNames(s) } } : { kind: "catalog" };
     if (relations.every(isCatalogRelation))
         return { kind: "catalog" };
     return { kind: "duckdb", sql: text, names: pgColumnNames(s) };
@@ -288,8 +329,12 @@ function setStatement(s) {
     if (s.kind === "VAR_RESET_ALL")
         return { kind: "set", name: "all", value: null, local: false, tag: "RESET" };
     // SET TRANSACTION is for the transaction alone, whichever way it is spelt; SET SESSION CHARACTERISTICS is the session's
-    if (s.kind === "VAR_SET_MULTI")
-        return { kind: "set", name, value: "", local: name === "transaction", tag: "SET" };
+    // its value is the isolation level it names, '' for none (`READ ONLY` alone): the listener remembers the level
+    if (s.kind === "VAR_SET_MULTI") {
+        const level = (s.args ?? []).map((a) => a.DefElem).find((d) => d?.defname === "transaction_isolation");
+        const value = level?.arg !== undefined && kindOf(level.arg) === "A_Const" ? constText(bodyOf(level.arg)) : "";
+        return { kind: "set", name, value, local: name === "transaction", tag: "SET" };
+    }
     if (s.kind !== "VAR_SET_VALUE")
         throw new Unsupported(`SET ${name}: ${String(s.kind)} is not supported`, s.location ?? null);
     const parts = (s.args ?? []).map((a) => {

package/dist/src/cli/background.js

@@ -17,6 +17,7 @@ import * as path from "node:path";
 import { spawn } from "node:child_process";
 import { fileURLToPath } from "node:url";
 import { runnable } from "../lib/runnable.js";
+import { LogReader, logPath } from "../lib/log-read.js";
 import { READY, STEP_MARK } from "./dev.js";
 import { liveRun } from "./runs.js";
 import { projectSource, startSettingRefusal, theProject } from "./project.js";
@@ -26,12 +27,13 @@ import { recordRefusal } from "./replication-mode.js";
 import { shownPath } from "./sources.js";
 import { endText, failed, jsonText, nextBlock } from "./command.js";
 import { historyDaysSetting } from "../duck/history.js";
+import { memoryProblems } from "../duck/store.js";
 import { DEV } from "./registry.js";
 import { UI } from "./ui.js";
 import { out as styleOut, stepRenderer } from "./style.js";
 import { postgresLink, withoutPlaceholders } from "./pg-url.js";
 const say = (m) => { process.stdout.write(`${m}\n`); };
-const MAIN = path.join(path.dirname(fileURLToPath(import.meta.url)), "main.ts");
+const MAIN = path.join(path.dirname(fileURLToPath(import.meta.url)), "mason.cjs");
 const SUPERVISE = path.join(path.dirname(fileURLToPath(import.meta.url)), "supervise.ts");
 /** The commands to run next, as `mason start` prints them once the run is up: one a line, what each does beside it
  *  (not `mason status | mason logs | mason stop` on one line). */
@@ -91,6 +93,11 @@ export async function devInBackground(args, o) {
     const days = historyDaysSetting(process.env);
     if (typeof days !== "number")
         return failed("start", { code: "refused", ...days }, { json: o.json });
+    // MASON_DUCKDB_MEMORY not an amount (src/duck/store.ts memoryProblems), by the run's own check, refused here as in the
+    // run, before anything is spawned or opened; a size under the floor only warns, in the run (memoryWarning)
+    const memory = memoryProblems(process.env)[0];
+    if (memory !== undefined)
+        return failed("start", { code: "refused", ...memory }, { json: o.json });
     // what the config records and the file does not say (src/cli/replication-mode.ts), in the run's order: settings first
     const unsaid = file === null ? null : recordRefusal(t.project.config, file, process.env, shown, "mason start");
     if (unsaid !== null)
@@ -107,12 +114,13 @@ export async function devInBackground(args, o) {
         return failed("start", listen, { json: o.json });
     fs.mkdirSync(home, { recursive: true, mode: 0o700 });
     // the run's terminal: its setup URL carries the run's token and it prints the API key, so only the developer reads it
-    const logFile = path.join(home, "dev.log");
+    const logFile = logPath(home, "start");
     const fd = fs.openSync(logFile, "a", 0o600);
     fs.chmodSync(logFile, 0o600);
     const from = fs.fstatSync(fd).size;
     fs.writeSync(fd, `\n--- mason start ${new Date().toISOString()} ---\n`);
-    // the run is started IN the project's root, so it finds the same project
+    // the run is started IN the project's root, so it finds the same project. Its env carries the size this process
+    // set (src/cli/pool-size.cjs), and the supervisor between them inherits it too: it never uses that many
     const run = [process.execPath, runnable(MAIN), "start", ...args.filter((a) => a !== "--json"), "--attach"];
     const child = spawn(process.execPath, [runnable(SUPERVISE), home, logFile, ...run], { detached: true, stdio: ["ignore", fd, fd], cwd: t.project.root, env: process.env });
     fs.closeSync(fd);
@@ -129,25 +137,15 @@ export async function devInBackground(args, o) {
     line(st.bold(`Starting Mason for ${t.project.config.name}`));
     const steps = stepRenderer({ line, live: !o.json && process.stdout.isTTY ? { write: (x) => { process.stdout.write(x); } } : null, style: st });
     let begun = false;
-    let offset = from;
-    let partial = "";
+    // the run's terminal from this start on, through the one reader: a line is had whole, masked (src/lib/log-read.ts)
+    const reader = new LogReader(logFile, { from });
     let postgres = null;
     const since = []; // the run's lines after its last step: a failure's reasons
     try {
         for (;;) {
-            const size = fs.statSync(logFile).size;
-            if (size > offset) {
-                const buf = Buffer.alloc(size - offset);
-                const r = fs.openSync(logFile, "r");
-                try {
-                    fs.readSync(r, buf, 0, buf.length, offset);
-                }
-                finally {
-                    fs.closeSync(r);
-                }
-                offset = size;
-                const lines = (partial + buf.toString()).split("\n");
-                partial = lines.pop() ?? "";
+            for (let text = reader.read(); text !== null; text = reader.read()) {
+                const lines = text.split("\n");
+                lines.pop(); // every line read ends in its newline
                 for (const l of lines) {
                     if (l.startsWith("--- mason start "))
                         continue;
@@ -206,7 +204,8 @@ export async function devInBackground(args, o) {
             if (exited !== null) {
                 if (steps.open() !== null)
                     steps.fail();
-                if (partial)
+                const partial = reader.rest(); // the line it was writing, unended
+                if (partial.trim())
                     since.push(partial.trim());
                 // the run's own last words are the reason (a wrong password), already shown line by line to a person
                 return failed("start", { message: `the run stopped before it was ready (exit ${exited})`, code: "failed", why: o.json && since.length ? since.join("\n") : undefined,

package/dist/src/cli/bridge-stdin.js

@@ -0,0 +1,101 @@
+/**
+ * THE BRIDGE'S STDIN, READ ON A THREAD OF ITS OWN, AND ITS WATCHDOG (src/cli/ui-ssh.ts `bridge`, `StdinByWorker`).
+ *
+ * `mason ui --bridge` ends when its stdin ends: the laptop's ssh gone. Heard on the main thread, that end waits on the
+ * main event loop, and a loop that is blocked (a synchronous call that never returns, a hung native call, a busy loop)
+ * never hears it: the process would live on, holding the run's tunnel, with nobody left who can end it. So this thread
+ * owns fd 0, on an event loop of its own (Node's docs, worker_threads: "each Worker has its own event loop"), forwards
+ * its bytes to the main thread, and on its end arms a timer here; if the process is still alive when it fires, the main
+ * thread did not end it, so this one does, by SIGKILL to the process (`process.kill`, which a worker may call: a signal
+ * is delivered by the system, not by a loop, and SIGKILL cannot be caught or delayed by anything the process runs).
+ *
+ * THE FLOW CONTROL IS THIS THREAD'S, BY CREDIT: it hands the main thread at most `windowBytes` it has not acknowledged
+ * (`ack`: consumed, and FrameLink not paused), and stops reading at that bound (`stream.pause()`, so the pipe fills
+ * and the laptop's writes wait, as they did on `process.stdin`). Nothing is read into memory past it, whatever the main
+ * thread does. A stopped reader cannot hear stdin's end, so the main thread says it is alive (`beat`, every
+ * `BEAT_MS`): a main thread that is only waiting (on the run's page taking what was sent) keeps beating and the reader
+ * waits with it; one silent for `exitMs` with the window full is STUCK, and this thread then reads on and DROPS what it
+ * reads, so that the end is heard and the watchdog can fire. The stream it fed is then broken, which the main thread is
+ * told (`dropped`): if it ever runs again it ends the bridge, exit 1, never reads past the hole.
+ *
+ * fd 0 is opened as Node opens `process.stdin` for its kind (Node's docs, process.stdin, net.Socket `fd`, tty.ReadStream,
+ * fs.createReadStream `fd`): a pipe or a socket (ssh's, `docker exec -i`'s) as a `net.Socket`, a terminal (`ssh -t`) as a
+ * `tty.ReadStream`, anything else (a file, /dev/null) as a file stream, which always reaches its end. A read error is an
+ * end too (a terminal hung up reads EIO). A stdin that cannot be opened at all is said, and ends the bridge.
+ */
+import * as fs from "node:fs";
+import * as net from "node:net";
+import * as tty from "node:tty";
+import { parentPort, workerData } from "node:worker_threads";
+const { exitMs, windowBytes } = workerData;
+if (parentPort === null)
+    throw new Error("src/cli/bridge-stdin.ts is a worker thread's script, started by src/cli/ui-ssh.ts");
+const port = parentPort;
+let over = false;
+/** Bytes handed to the main thread that it has not acknowledged. */
+let owed = 0;
+let lastHeard = Date.now();
+let dropping = false;
+/** The end, once: said to the main thread, then the watchdog armed. */
+function finish(why, failed = false) {
+    if (over)
+        return;
+    over = true;
+    port.postMessage({ end: why, failed });
+    setTimeout(() => {
+        try {
+            fs.writeSync(2, `mason ui --bridge: its stdin closed ${exitMs / 1000} s ago and it had not ended (its main thread is blocked): ended now, and the tunnel with it\n`);
+        }
+        catch (e) {
+            void e; /* stderr gone too, as the laptop's ssh is: the kill below is what ends the process */
+        }
+        process.kill(process.pid, "SIGKILL");
+    }, exitMs);
+}
+function open() {
+    if (tty.isatty(0))
+        return new tty.ReadStream(0);
+    const kind = fs.fstatSync(0);
+    if (kind.isFIFO() || kind.isSocket())
+        return new net.Socket({ fd: 0, readable: true, writable: false });
+    return fs.createReadStream("", { fd: 0, autoClose: false });
+}
+let input = null;
+try {
+    input = open();
+}
+catch (e) {
+    finish(`its stdin cannot be read: ${e.message}`, true);
+}
+if (input !== null) {
+    const stream = input;
+    stream.on("data", (b) => {
+        if (dropping)
+            return;
+        const copy = Uint8Array.from(b);
+        port.postMessage({ data: copy }, [copy.buffer]);
+        owed += b.length;
+        if (owed >= windowBytes)
+            stream.pause();
+    });
+    stream.once("end", () => { finish(null); });
+    stream.once("error", (e) => { finish(`its stdin: ${e.message}`); });
+    stream.once("close", () => { finish(null); });
+    port.on("message", (m) => {
+        lastHeard = Date.now();
+        if (!("ack" in m) || dropping)
+            return;
+        owed -= m.ack;
+        if (owed < windowBytes && stream.isPaused())
+            stream.resume();
+    });
+    // a main thread silent for the bound while the window is full is stuck: read on, dropping, so the end is heard
+    const look = setInterval(() => {
+        if (dropping || over || owed < windowBytes || Date.now() - lastHeard < exitMs)
+            return;
+        dropping = true;
+        clearInterval(look);
+        port.postMessage({ dropped: owed });
+        stream.resume();
+    }, Math.min(250, exitMs));
+}