@mason-data/cli 0.7.0 → 0.8.0

package/dist/src/cli/supervise.js

@@ -42,9 +42,20 @@ export function readRestarts(home) {
     const v = JSON.parse(fs.readFileSync(file, "utf8"));
     return Array.isArray(v.restarts) ? v.restarts : [];
 }
+export function restartsFact(home, read = () => readRestarts(home)) {
+    let r;
+    try {
+        r = read();
+    }
+    catch (e) {
+        return { error: `${path.join(home, RESTARTS_FILE)} could not be read: ${e.message}` };
+    }
+    const last = r[r.length - 1];
+    return { count: r.length, last: last === undefined ? null : { at: last.at, why: last.reason, upSeconds: last.upSeconds } };
+}
 /** Why a run died, in the words a person reads: V8's own fatal line when it printed one */
 export function crashReason(code, signal, fatal) {
-    // Mason's own fatal line (src/cli/dev.ts: the copy's thread stopped) is said as Mason said it
+    // Mason's own fatal line (src/cli/dev.ts: the engine's thread stopped) is said as Mason said it
     if (fatal !== null)
         return /heap out of memory/i.test(fatal) && !fatal.startsWith("Mason: ") ? "it ran out of JavaScript heap (V8: heap out of memory)"
             : fatal.startsWith("Mason: ") ? fatal.slice("Mason: ".length) : `V8: ${fatal}`;

package/dist/src/cli/sync.js

@@ -1,9 +1,10 @@
 /**
  * `mason sync <table>…`: Mason's poll of exactly these tables, now — named for what it does to the copy
- * (made current, incrementally), never for the mechanism (`poll` was). The passes poll every added table but one over
- * the poll's row limit (src/duck/sync.ts POLL_ROW_LIMIT: each poll scans its table whole on the database), which is
- * copied and served as it was copied; this reads what changed in it since, when asked. The run's `POST /api/sync`
- * (src/cli/run-server.ts) answers it, through the sync's own poll (`Sync.pollNow`).
+ * (made current, incrementally), never for the mechanism (`poll` was). A table whose sources-file entry says `sync:
+ * off` (the default: src/lib/table-sync.ts) is read again only so, and one set to `poll <interval>` is polled now
+ * whatever its interval says, its next poll an interval from this one. A table whose sync is `replication` is not
+ * polled: its answer is where the stream is, beside the polled ones'. The run's `POST /api/sync` (src/cli/run-server.ts) answers it, through the sync's own
+ * poll (`Sync.pollNow`).
  */
 import { failed, jsonText, parseCommand, reportText } from "./command.js";
 import { out as styleOut } from "./style.js";
@@ -11,30 +12,55 @@ import { callRun } from "./control.js";
 import { reach } from "./state.js";
 import { findTable, guarded, readState, tableArg } from "./commands.js";
 import { didYouMean } from "../lib/did-you-mean.js";
-import { POLL_ROW_LIMIT } from "../duck/sync.js";
+import { syncedText, syncWords } from "../lib/table-sync.js";
 const out = (m) => { process.stdout.write(`${m}\n`); };
 const n = (x) => x.toLocaleString("en-US");
 export const SYNC_USAGE = `usage: mason sync <table>… [--json]
 
-  Makes Mason's copy of these tables current now: it reads what changed in your database since Mason last read them —
-  the rows inserted or updated, and the ones deleted. Mason syncs every added table by itself every few seconds, but
-  a table over ${n(POLL_ROW_LIMIT)} rows is copied and served without that (each sync scans a table whole on your
-  database): mason status says which, and this syncs one when you need it current. mason start --replication keeps
-  every table current from the database's change log, whatever its size.
+  Brings these synced tables up to date now: it reads what changed in your database since Mason last read them —
+  the rows inserted or updated, and the ones deleted — in one transaction, as a poll does (each one scans its table
+  whole on your database). After its first sync, a table is read again as its sync: in the sources file says: off
+  (the default) only when you run this — before a report, from cron, at the end of a deploy — and poll <interval> by
+  itself every interval, which this does now too, whatever the interval says. mason status says how each table is
+  kept current and when it was last synced. A table whose sync: is replication is not polled: it answers where the
+  stream of the database's change log is, beside the tables it polls.
 
     mason sync orders
     mason sync orders public.events
 
-  --json   as data: { syncedAt, tables: [{ table, changed, deleted, rows, ms }], next }
+  --json   as data: { syncedAt, tables: [{ table, changed, deleted, rows, ms }] (the tables polled), streamed (the
+           tables whose sync: is replication, not polled: [{ table, position, state, why }]), next }
 `;
 export const SYNC = {
     needs: "run",
     name: "sync",
-    summary: "make the copy of these tables current now: mason sync orders (a table over the sync's row limit is synced only so)",
+    summary: "bring these synced tables up to date now: mason sync orders (a table whose sync is off is read again only so)",
     usage: SYNC_USAGE,
     options: { json: { type: "boolean" } },
-    json: ["syncedAt", "tables", "next"],
+    json: ["syncedAt", "tables", "streamed", "next"],
 };
+/**
+ * WHAT A SYNC DID, as `mason sync` prints it from `POST /api/sync`'s answer (the web page's Sync button says the same,
+ * src/web/pages/tables.js; tests/web-page.test.ts holds the two equal): the rows written and deleted, a line per table,
+ * and what happens next to each table synced (`syncedTips`: an `off` one stays as of this sync). A table whose sync is
+ * replication was not polled: its line says where the stream is (`streamed`), and a sync of those alone has nothing to sync.
+ */
+export function syncedReport(b, picked, tables) {
+    const streamed = b.streamed ?? [];
+    const streamedRows = streamed.map((t) => [t.table, `${t.state}${t.why ? `: ${t.why}` : ""} · the stream has applied the database's changes through ${t.position}`]);
+    if (b.tables.length === 0 && streamed.length > 0) {
+        return { headline: `Nothing to sync: logical replication keeps ${picked.join(", ")} current.`, details: streamedRows, tips: [] };
+    }
+    const changed = b.tables.reduce((a, t) => a + t.changed, 0), deleted = b.tables.reduce((a, t) => a + t.deleted, 0);
+    const streamedNames = new Set(streamed.map((t) => t.table));
+    const polled = picked.filter((t) => !streamedNames.has(t));
+    return {
+        headline: `Synced ${polled.join(", ")}: ${n(changed)} row(s) written, ${n(deleted)} deleted${streamedRows.length ? `; logical replication keeps ${streamedRows.map((r) => r[0]).join(", ")} current` : ""}.`,
+        details: [...b.tables.map((t) => [t.table, syncedText(t)]), ...streamedRows],
+        // what happens next to each: an `off` table stays as of now, a polled one is polled again after its interval
+        tips: syncedTips(tables),
+    };
+}
 export async function sync(argv) {
     const f = parseCommand(SYNC, argv);
     if (typeof f === "number")
@@ -62,7 +88,7 @@ export async function sync(argv) {
             if ("message" in db && db.code === "usage")
                 return failed("sync", db, { json });
             if (!("message" in db)) {
-                return failed("sync", { message: `${db.table} is not added, so Mason has no copy of it to sync`, code: "not_found", fix: `mason add ${tableArg(db.table)}`,
+                return failed("sync", { message: `${db.table} is not added, so Mason has not synced it`, code: "not_found", fix: `mason add ${tableArg(db.table)}`,
                     next: [["mason status", "the added tables"]] }, { json });
             }
             return failed("sync", { message: `there is no added table ${JSON.stringify(name)}${didYouMean(name, added.flatMap((x) => [x.table, tableArg(x.table)]))}`, code: "not_found",
@@ -73,22 +99,26 @@ export async function sync(argv) {
         if (r.status !== 200) {
             const code = typeof r.body.code === "string" ? r.body.code : r.status === 404 ? "not_found" : "failed";
             return failed("sync", { message: `${picked.join(", ")} ${picked.length === 1 ? "was" : "were"} not synced`, code, why: String(r.body.error),
-                next: [["mason status", "each table's copy"], ["mason logs", "the run's log"]] }, { json });
+                next: [["mason status", "each synced table"], ["mason logs", "the run's log"]] }, { json });
         }
         const b = r.body;
-        const unsynced = added.filter((t) => picked.includes(t.table) && t.unsynced);
-        const next = [[`mason sync ${names.join(" ")}`, "sync again, whenever you need it current"], ["mason status", "each table's copy, and which the sync leaves out"]];
+        const next = [[`mason sync ${names.join(" ")}`, "sync again, whenever you need it current"], ["mason status", "how each table is kept current, and when it was last synced"]];
         if (json) {
-            out(jsonText(SYNC, { syncedAt: b.at, tables: b.tables }, next));
+            out(jsonText(SYNC, { syncedAt: b.at, tables: b.tables, ...(b.streamed ? { streamed: b.streamed } : {}) }, next));
             return 0;
         }
-        const changed = b.tables.reduce((a, t) => a + t.changed, 0), deleted = b.tables.reduce((a, t) => a + t.deleted, 0);
-        out(reportText({
-            headline: `Synced ${picked.join(", ")}: ${n(changed)} row(s) written, ${n(deleted)} deleted.`,
-            details: b.tables.map((t) => [t.table, `${n(t.changed)} changed, ${n(t.deleted)} deleted, ${n(t.rows)} rows now · ${n(t.ms)} ms`]),
-            tips: unsynced.map((t) => `${t.table} is ${t.unsynced.reason}: it stays as of this sync until the next mason sync (mason start --replication keeps every table current)`),
-            next,
-        }, styleOut()));
+        out(reportText({ ...syncedReport(b, picked, added.filter((t) => picked.includes(t.table))), next }, styleOut()));
         return 0;
     });
 }
+/** What happens to each table synced after now, by its sync (src/duck/sync.ts KeptCurrent): an `off` table stays as of
+ *  this sync until the next one, a polled one is polled again after its interval. */
+export function syncedTips(tables) {
+    return tables.flatMap((t) => {
+        const k = t.kept;
+        if (k?.by !== "poll")
+            return [];
+        return [k.sync.mode === "off" ? `${t.table}'s sync is off: it stays as of this sync until the next mason sync ${tableArg(t.table)}`
+                : `${t.table} is ${syncWords(k.sync)}: its next poll is an interval from this one`];
+    });
+}

package/dist/src/cli/tunnel.js

@@ -0,0 +1,200 @@
+/**
+ * THE TUNNEL OF `mason share`: a client of a free service that forwards raw TCP, run as a child process and pointed at the
+ * share's socket (src/cli/share-run.ts). What crosses is TLS between the BI tool and Mason, so a service that only
+ * forwards reads nothing; a hostile one could sit in the handshake (docs/POSTGRES-API.md says so). Which one is `--via`:
+ * Pinggy over the `ssh` every machine has, or `bore`, a binary to install. The address is the client's OUTPUT, a format
+ * Mason does not own, so reading it is held three ways: a time limit (`timeoutMs`), a shape it must have
+ * (`tunnelAddress`), and a probe of the address it gave (`probeShare`): the share's own certificate must answer there.
+ *
+ * `ssh` NEVER PROMPTS, because nothing stands behind a prompt (an agent, a pipe, a terminal that went on to something
+ * else): it runs in a session of its own with no terminal (`detached`), stdin a pipe this process holds. The first
+ * connection's host key is taken and kept in `.mason/share-known-hosts` (`accept-new`: a key that CHANGES later fails, said
+ * by the file's name), never the developer's own `~/.ssh`; their keys, agent and config are left out (`-F`, no
+ * `PubkeyAuthentication`), so nothing can ask for a passphrase or a touch and nothing of theirs is offered to a
+ * service. Pinggy's free route asks for a password that is empty (its docs: "just press enter"), so a password
+ * question is answered by `true`, which prints nothing (`SSH_ASKPASS`, forced): that is the one answer any question gets.
+ *
+ * A PINGGY TOKEN NEVER REACHES A COMMAND LINE: `ps` shows every process's arguments to every user, and the service takes the
+ * token as the ssh user. So the user is written in a config file of ours, 0600 in `.mason/`, named by `-F`, and
+ * removed as soon as the tunnel is up (ssh has read it), or when it fails. `MASON_PINGGY_TOKEN` is read from the
+ * shell or `.env`, never a flag, and is masked in whatever of the client's output a failure repeats.
+ */
+import { spawn } from "node:child_process";
+import * as fs from "node:fs";
+import * as net from "node:net";
+import * as path from "node:path";
+import * as tls from "node:tls";
+import { stripVTControlCharacters } from "node:util";
+export const TUNNELS = ["pinggy", "bore"];
+/** A free tunnel's limit where its client does not say one: Pinggy's, measured in docs/DESIGN.md "mason share". */
+const FREE_MINUTES = 60;
+/** How much of the client's output is kept to read an address from, and how many lines a failure repeats. */
+const KEPT_BYTES = 64 * 1024;
+const SAID_LINES = 3;
+const STOP_GRACE_MS = 2_000;
+/** How long the address is given to answer: long enough for a relay on another continent, short enough to wait for. */
+export const PROBE_MS = 8_000;
+/** The ssh user and host: a free tunnel, or a token's (`<token>+tcp@`, Pinggy's own form). */
+const pinggyTarget = (token) => token === null ? { user: "tcp", host: "free.pinggy.io" } : { user: `${token}+tcp`, host: "pro.pinggy.io" };
+/** A token as it may stand in a config file: anything else would be a line of it. */
+export const TOKEN_RE = /^[A-Za-z0-9._~-]+$/;
+const quoted = (file) => `"${file.replaceAll("%", "%%")}"`;
+/** The ssh config of a tunnel (the header says why each line): every option here, so the command line carries none. */
+export function pinggyConfig(o) {
+    const t = pinggyTarget(o.token);
+    return `Host mason-share
+  HostName ${t.host}
+  Port 443
+  User ${t.user}
+  StrictHostKeyChecking accept-new
+  UserKnownHostsFile ${quoted(o.knownHosts)}
+  GlobalKnownHostsFile none
+  PubkeyAuthentication no
+  IdentityAgent none
+  NumberOfPasswordPrompts 1
+  ExitOnForwardFailure yes
+  ServerAliveInterval 30
+  ServerAliveCountMax 3
+  ConnectTimeout 15
+  ConnectionAttempts 1
+  RequestTTY no
+  LogLevel ERROR
+`;
+}
+export function tunnelPlan(via, o) {
+    if (via === "bore") {
+        return { via, bin: "bore", args: ["local", String(o.port), "--local-host", "127.0.0.1", "--to", "bore.pub"], env: process.env, install: "cargo install bore-cli, or brew install bore-cli",
+            reconnects: false, free: false, token: null, file: null };
+    }
+    const file = path.join(o.home, `share-tunnel-${process.pid}.ssh`);
+    return { via, bin: "ssh", args: ["-F", file, "-T", `-R0:127.0.0.1:${o.port}`, "mason-share"],
+        env: { ...process.env, SSH_ASKPASS: "true", SSH_ASKPASS_REQUIRE: "force", DISPLAY: process.env.DISPLAY ?? ":0" }, install: "the OpenSSH client (it comes with macOS and most Linux; apt install openssh-client)",
+        reconnects: o.token !== null, free: o.token === null, token: o.token,
+        file: { path: file, text: pinggyConfig({ token: o.token, knownHosts: path.join(o.home, "share-known-hosts") }) } };
+}
+/** A `share-tunnel-<pid>.ssh` a `kill -9` left behind (it may hold a token): removed unless its process is alive. */
+export function sweepTunnelFiles(home, isAlive) {
+    for (const name of fs.readdirSync(home)) {
+        const m = /^share-tunnel-(\d+)\.ssh$/.exec(name);
+        if (m !== null && !isAlive(Number(m[1])))
+            fs.rmSync(path.join(home, name), { force: true });
+    }
+}
+const PINGGY_ADDRESS = /\btcp:\/\/([a-z0-9][a-z0-9.-]*):(\d{1,5})(?![\d.])/i;
+const BORE_ADDRESS = /\blistening at ([a-z0-9][a-z0-9.-]*):(\d{1,5})(?![\d.])/i;
+/** The public address in what a client printed so far, or null: a shape, and a real port. */
+export function tunnelAddress(via, output) {
+    const m = (via === "pinggy" ? PINGGY_ADDRESS : BORE_ADDRESS).exec(stripVTControlCharacters(output));
+    if (m === null)
+        return null;
+    const port = Number(m[2]);
+    return port >= 1 && port <= 65_535 ? { host: m[1].toLowerCase(), port } : null;
+}
+/** When a free tunnel ends: what its client says, else its known limit; null when it has none. */
+export function tunnelEnds(plan, output, now) {
+    if (!plan.free)
+        return null;
+    const said = /expires?\s+in\s+(\d+)\s*min/i.exec(stripVTControlCharacters(output));
+    return new Date(now + (said === null ? FREE_MINUTES : Number(said[1])) * 60_000).toISOString();
+}
+/** The last lines a client printed, on one line, its token masked: what a failure repeats. */
+export function lastLines(output, token) {
+    const lines = stripVTControlCharacters(output).split(/[\r\n]+/).map((l) => l.trim()).filter((l) => l !== "");
+    const text = lines.slice(-SAID_LINES).join(" / ");
+    return token === null ? text : text.replaceAll(token, "<token>");
+}
+/** Starts a tunnel client and reads its address: the tunnel, or why not. `onPid`: as soon as there is a process. */
+export function startTunnel(plan, o) {
+    return new Promise((resolve) => {
+        const forget = () => { if (plan.file !== null)
+            fs.rmSync(plan.file.path, { force: true }); };
+        if (plan.file !== null)
+            fs.writeFileSync(plan.file.path, plan.file.text, { mode: 0o600 });
+        // detached: a session of its own, so no terminal to prompt on, and Ctrl-C reaches this process alone, which ends it.
+        // stdin a pipe held open: the end of stdin would end the session the service prints on
+        const child = spawn(plan.bin, plan.args, { stdio: ["pipe", "pipe", "pipe"], env: plan.env, detached: true });
+        let output = "", settled = false, finished = () => undefined;
+        const ended = new Promise((r) => { finished = r; });
+        const stop = () => {
+            if (child.exitCode !== null || child.signalCode !== null)
+                return;
+            child.kill("SIGTERM");
+            setTimeout(() => { if (child.exitCode === null && child.signalCode === null)
+                child.kill("SIGKILL"); }, STOP_GRACE_MS).unref();
+        };
+        const settle = (v) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            forget();
+            resolve(v);
+        };
+        const timer = setTimeout(() => {
+            stop();
+            const said = lastLines(output, plan.token);
+            settle({ message: `the tunnel gave no address in ${Math.round(o.timeoutMs / 1000)} s`, code: "failed", why: said === "" ? `${plan.bin} printed nothing` : `${plan.bin} printed: ${said}`,
+                fix: `its output is the service's, and may have changed: this Mason cannot read it${plan.via === "pinggy" ? ", and --via bore is another tunnel" : ""}. Nothing was left open` });
+        }, o.timeoutMs);
+        const read = (b) => {
+            output = (output + b.toString("utf8")).slice(-KEPT_BYTES);
+            if (settled)
+                return;
+            const address = tunnelAddress(plan.via, output);
+            if (address === null)
+                return;
+            settle({ ...address, endsAt: tunnelEnds(plan, output, Date.now()), pid: child.pid, ended, stop });
+        };
+        child.stdout.on("data", read);
+        child.stderr.on("data", read);
+        // the end of stdin would be a command's end, and EPIPE from a client gone is that end, said by its exit
+        child.stdin.on("error", () => undefined);
+        child.on("spawn", () => o.onPid?.(child.pid));
+        child.on("error", (e) => {
+            if (e.code !== "ENOENT") {
+                settle({ message: `${plan.bin} could not be started: ${e.message}`, code: "failed" });
+                return;
+            }
+            settle({ message: `${plan.bin} is not on this machine's PATH, and --via ${plan.via} needs it`, code: "not_found", fix: `install ${plan.install}${plan.via === "pinggy" ? ", or use --via bore" : ", or use --via pinggy, which needs only ssh"}`,
+                next: [[`mason share --via ${plan.via === "pinggy" ? "bore" : "pinggy"}`, "the other tunnel"]] });
+        });
+        const over = (code, signal) => {
+            const said = lastLines(output, plan.token);
+            const how = signal !== null ? `it was ended (${signal})` : `${plan.bin} exited ${code}`;
+            finished(said === "" ? how : `${how}: ${said}`);
+            settle({ message: "the tunnel ended before it gave an address", code: "failed", why: said === "" ? how : `${how}: ${said}`,
+                fix: plan.via === "pinggy" && /REMOTE HOST IDENTIFICATION|Host key verification failed/i.test(said) ? "the service's host key changed: remove .mason/share-known-hosts if you expected that" : "is this machine online? Port 443 outbound is all it needs" });
+        };
+        // 'close' waits for its pipes, which a grandchild can hold for ever: 'exit' ends the wait
+        let code = null, signal = null;
+        child.on("exit", (c, s) => { code = c; signal = s; setTimeout(() => over(c, s), 200).unref(); });
+        child.on("close", () => over(code, signal));
+    });
+}
+/**
+ * WHETHER THE ADDRESS IS THIS SHARE: an SSLRequest and a handshake through the tunnel, its certificate read, no login.
+ * `other`: something else answered there, a wrong address or something in the way. `unknown`: nothing answered in time
+ * (a firewall may keep this machine from a port the BI tool can reach): no verdict, and said so.
+ */
+export function probeShare(address, fingerprint, timeoutMs = PROBE_MS) {
+    return new Promise((resolve) => {
+        const socket = net.connect(address.port, address.host);
+        let secure = null;
+        // the TLS socket owns the connection once it wraps it: destroying the one under it segfaults Node (measured, 24.18)
+        const answer = (v) => { clearTimeout(timer); (secure ?? socket).destroy(); resolve(v); };
+        const timer = setTimeout(() => answer("unknown"), timeoutMs);
+        socket.once("error", () => answer("unknown"));
+        socket.once("close", () => answer("unknown"));
+        socket.once("connect", () => socket.write(Buffer.from([0, 0, 0, 8, 0x04, 0xd2, 0x16, 0x2f])));
+        socket.once("data", (b) => {
+            if (b[0] !== 0x53 /* S */) {
+                answer("other");
+                return;
+            }
+            socket.removeAllListeners("close");
+            secure = tls.connect({ socket, rejectUnauthorized: false, ALPNProtocols: ["postgresql"] });
+            secure.once("error", () => answer("unknown"));
+            secure.once("secureConnect", () => answer(secure?.getPeerX509Certificate()?.fingerprint256 === fingerprint ? "same" : "other"));
+        });
+    });
+}

package/dist/src/cli/ui.js

@@ -1,6 +1,7 @@
 /**
- * `mason ui` — THE WEB PAGE OF THE RUNNING MASON, IN THE BROWSER (src/web/, docs/WEB-UI.md): what `mason status` says,
- * refreshed while the tab is seen, and the users of `mason user`. It prints the page's URL WITH the run's token (src/cli/security.ts: the first request trades it for an HttpOnly cookie and is redirected to a URL without
+ * `mason ui` — THE WEB PAGE OF THE RUNNING MASON, IN THE BROWSER (src/web/, docs/WEB-UI.md): the Overview (what `mason
+ * status` says), the Tables (`mason tables`, `add`, `remove`, `sync`), the Users (`mason user`) and Observability (`mason
+ * logs`, `mason usage`), refreshed while the tab is seen. It prints the page's URL WITH the run's token (src/cli/security.ts: the first request trades it for an HttpOnly cookie and is redirected to a URL without
  * it) and opens the browser. The reader is the person at this terminal, whom `mason user add` already shows a
  * password once; the token is never written anywhere else — not in `mason start`'s output (its log is a file), not in
  * `mason status`, which prints the URL without it.
@@ -16,12 +17,14 @@ const out = (m) => { process.stdout.write(`${m}\n`); };
 export const UI = {
     needs: "run",
     name: "ui",
-    summary: "the status page and the users, in your browser",
+    summary: "the status, tables, users and statements, in your browser",
     usage: `usage: mason ui [--no-open] [--json]
 
-  Opens the running Mason's web page in your browser: what mason status says (the database, each table's copy and
-  its progress, the slow and refused statements, whether it is ready), refreshed every 2 s while the tab is open, and
-  the users of mason user — add one (its password shown once, with the URL to paste) or remove one.
+  Opens the running Mason's web page in your browser, refreshed while the tab is open: the Overview (what mason status
+  says: whether the synced data is ready, the database, the sync, the statements and the recent errors); the Tables (add,
+  remove or sync one, as mason add, remove and sync do); the Users of mason user (add one, its password shown once with
+  the URL to paste, or remove one); and Observability: the statements running and that ran (mason logs --requests,
+  cancel one), usage over time (mason usage), and the sync's events (mason logs --events).
 
   The URL carries this run's token: the page trades it for a cookie at once and drops it from the address
   bar. It is printed for you alone, like a user's password: paste it nowhere else. Every mason start makes a new

package/dist/src/cli/usage.js

@@ -0,0 +1,128 @@
+/**
+ * `mason usage`: what the run was and did over time — a sample a minute (src/duck/usage.ts) of the copy's size and
+ * rows, the copies failed, how old the stalest copy is, the statements and their duration, the stream's lag and the
+ * memory — and each copy's own size. The run's `GET /api/usage` (src/cli/run-server.ts) answers it; `--json` prints that
+ * answer as it is, so an agent and a chart read the same samples a person reads here.
+ */
+import { failed, jsonText, parseCommand, endText } from "./command.js";
+import { out as styleOut } from "./style.js";
+import { callRun } from "./control.js";
+import { reach } from "./state.js";
+import { fileSize, guarded, runFailure } from "./commands.js";
+import { timeOf } from "./request-filter.js";
+import { duration, msText } from "./readiness.js";
+import { SIZE_EVERY_MS } from "../duck/usage.js";
+const out = (m) => { process.stdout.write(`${m}\n`); };
+const n = (x) => x.toLocaleString("en-US");
+export const USAGE_USAGE = `usage: mason usage [--since <when>] [--until <when>] [--step <span>] [--json]
+
+  What Mason was and did over time, from a sample it takes every minute and keeps MASON_HISTORY_DAYS days
+  (default 7): a line each span — the statements of the Postgres API (how many, how many were refused, failed or
+  cancelled — the errors column —, the completed ones' p95 duration, the rows sent), the synced data's size and rows, the
+  tables whose sync is failing, how old the stalest table was, the stream's lag (a table under sync: replication), and
+  the memory of the process and of the engine against its limit (MASON_DUCKDB_MEMORY). Then each synced table's own size, measured every
+  10 minutes. A span sums its statements and shows each level at its highest. Read through the running Mason.
+
+    mason usage                        the last hour, 5 minutes a line
+    mason usage --since 7d --step 1h   the last week, an hour a line
+    mason usage --since 2026-10-28T09:00:00Z --until 2026-10-28T10:00:00Z --step 0
+
+  --since <when>   from when: an age (90s, 30m, 6h, 7d) or an ISO 8601 date or time with its zone; default 1h
+  --until <when>   until when, the same way; default now
+  --step <span>    a line per span (1m to the days kept; default 5m), or 0 for every sample as it was taken
+  --json           as data: { since, until, stepSeconds (null: every sample), sampleSeconds, keptDays, samples: [{ at
+                   (with a step, the span's start), seconds, copyBytes, copyRows, tablesAdded, tablesCopied,
+                   tablesFailed, oldestSyncSeconds, statements, statementsRefused, statementsFailed, statementsCancelled,
+                   rowsSent, bytesSent, durationP50Ms, durationP95Ms, streamLagBytes, streamLagSeconds, rssBytes,
+                   heapUsedBytes, copyMemoryBytes, copyMemoryLimitBytes, slotWalHeldBytes (replication: the WAL the
+                   slot holds on the database), diskFreeBytes (free where the synced data is; with a step, the lowest),
+                   sessionsOpen (the Postgres API's sessions logged in) }],
+                   copySizes: [{ table, bytes, at }], next }
+`;
+export const USAGE = {
+    needs: "run",
+    name: "usage",
+    summary: "what Mason did over time: statements, the synced data's size and age, memory (a sample a minute; --since 7d --step 1h)",
+    usage: USAGE_USAGE,
+    options: { since: { type: "string" }, until: { type: "string" }, step: { type: "string" }, json: { type: "boolean" } },
+    json: ["since", "until", "stepSeconds", "sampleSeconds", "keptDays", "samples", "copySizes", "next"],
+};
+const UNIT = { s: 1, m: 60, h: 3600, d: 86_400 };
+/** A span as the options write it (90s, 30m, 6h, 7d; 0), in seconds, or null when it is not one. */
+export function spanSeconds(text) {
+    if (text.trim() === "0")
+        return 0;
+    const m = /^\s*(\d+)\s*([smhd])\s*$/.exec(text);
+    return m ? Number(m[1]) * UNIT[m[2]] : null;
+}
+/** One sample on one line: the time, the statements, those refused, failed or cancelled, p95, rows sent, the copy, its rows, the stalest copy, the lag, the memory. */
+export function sampleLine(s, withLag) {
+    const time = s.at.slice(0, 16).replace("T", " ");
+    const p95 = s.durationP95Ms === null ? "–" : msText(s.durationP95Ms);
+    const bad = s.statementsRefused + s.statementsFailed + s.statementsCancelled;
+    const age = s.oldestSyncSeconds === null ? "–" : duration(s.oldestSyncSeconds);
+    const lag = !withLag ? "" : `  ${(s.streamLagSeconds === null ? "–" : duration(s.streamLagSeconds)).padStart(9)}`;
+    const memory = `${fileSize(s.rssBytes)}${s.copyMemoryBytes === null ? "" : ` (engine ${fileSize(s.copyMemoryBytes)} of ${fileSize(s.copyMemoryLimitBytes)})`}`;
+    return `${time}  ${n(s.statements).padStart(10)}  ${(bad === 0 ? "" : n(bad)).padStart(7)}  ${p95.padStart(10)}  ${n(s.rowsSent).padStart(11)}  ` +
+        `${(s.copyBytes === null ? "–" : fileSize(s.copyBytes)).padStart(10)}  ${n(s.copyRows).padStart(13)}  ${(s.tablesFailed === 0 ? "" : n(s.tablesFailed)).padStart(7)}  ` +
+        `${age.padStart(10)}${lag}  ${memory}`;
+}
+/** The heading over `sampleLine`'s columns. */
+export function sampleHeading(withLag) {
+    return `${"time (UTC)".padEnd(16)}  ${"statements".padStart(10)}  ${"errors".padStart(7)}  ${"p95".padStart(10)}  ${"rows sent".padStart(11)}  ` +
+        `${"size".padStart(10)}  ${"rows".padStart(13)}  ${"failing".padStart(7)}  ${"oldest".padStart(10)}${withLag ? `  ${"lag".padStart(9)}` : ""}  memory`;
+}
+export async function usage(argv, now = Date.now()) {
+    const f = parseCommand(USAGE, argv);
+    if (typeof f === "number")
+        return f;
+    const v = f.values;
+    const json = v.json === true;
+    if (f.positionals.length)
+        return failed("usage", { message: `it takes no ${f.positionals.length === 1 ? "argument" : "arguments"}: ${f.positionals.join(" ")}`, code: "usage", usage: USAGE_USAGE }, { json });
+    // the one time grammar (src/cli/request-filter.ts timeOf): an age, or an ISO 8601 date or time WITH its zone
+    const since = timeOf("--since", v.since ?? "1h", now), until = v.until === undefined ? new Date(now).toISOString() : timeOf("--until", v.until, now);
+    for (const t of [since, until])
+        if (typeof t !== "string")
+            return failed("usage", { ...t, usage: USAGE_USAGE }, { json });
+    const step = spanSeconds(v.step ?? "5m");
+    if (step === null)
+        return failed("usage", { message: `--step is a span (1m, 5m, 1h, 1d) or 0 for every sample, not ${JSON.stringify(v.step)}`, code: "usage", usage: USAGE_USAGE }, { json });
+    const w = await reach(USAGE, json);
+    if (w === null || !("run" in w))
+        return 1;
+    const run = w.run;
+    return guarded("usage", json, async () => {
+        const q = new URLSearchParams({ since: since, until: until, ...(step === 0 ? {} : { step: String(step) }) });
+        const r = await callRun(run, "GET", `/api/usage?${q}`);
+        // the run's refusal (a step past the days kept, a since after until) in its own words and code
+        if (r.status !== 200)
+            return failed("usage", runFailure(r, [["mason usage --help", "the spans and times it takes"], ["mason logs", "the run's log"]]), { json });
+        const b = r.body;
+        const next = [["mason usage --since 7d --step 1h", "the week, an hour a line"], ["mason status", "where everything is now"]];
+        if (json) {
+            out(jsonText(USAGE, b, next));
+            return 0;
+        }
+        const st = styleOut();
+        const span = b.stepSeconds === null ? "every sample" : `${duration(b.stepSeconds)} a line`;
+        out(st.bold(`Usage from ${b.since.slice(0, 16).replace("T", " ")} to ${b.until.slice(0, 16).replace("T", " ")} UTC, ${span} (a sample every ${duration(b.sampleSeconds)}, kept ${b.keptDays} day${b.keptDays === 1 ? "" : "s"}):`));
+        if (!b.samples.length)
+            out(st.dim("  no sample in this time: Mason takes the first a minute after it starts"));
+        else {
+            const withLag = b.samples.some((s) => s.streamLagBytes !== null || s.streamLagSeconds !== null);
+            out(st.dim(sampleHeading(withLag)));
+            for (const s of b.samples)
+                out(sampleLine(s, withLag));
+        }
+        if (b.copySizes.length) {
+            out("");
+            out(st.bold(`Each synced table's own size (measured every ${duration(SIZE_EVERY_MS / 1000)}):`));
+            const wide = Math.max(...b.copySizes.map((c) => c.table.length));
+            for (const c of b.copySizes)
+                out(`  ${c.table.padEnd(wide)}  ${fileSize(c.bytes).padStart(10)}  ${st.dim(`at ${c.at.slice(11, 16)} UTC`)}`);
+        }
+        out(endText({ next }, st));
+        return 0;
+    });
+}