@mason-data/cli 0.0.0-stage → 0.6.2

package/dist/src/cdc/source.js

@@ -0,0 +1,65 @@
+/**
+ * THE REPLICATION SLOT, and the names a run replicates through (src/duck/sync.ts, src/cli/replication-mode.ts). The
+ * slot is the one object Mason creates on the source (its user needs the REPLICATION attribute); a publication is read,
+ * never created there, unless it is Mason's own (src/duck/publication.ts; CLAUDE.md "We are read-only on a customer's
+ * Postgres").
+ */
+import { PgConnection, parseLsn, pgIdent, pgLiteral } from "./pg-wire.js";
+import { parsePort } from "../lib/port.js";
+/**
+ * The publication and slot a project replicates through: `PG_PUBLICATION` / `PG_SLOT_NAME`, else Mason's own
+ * `mason_<name>`. A publication of Mason's own name is one Mason may create; any other is the database owner's and is
+ * only ever read.
+ */
+export function replicationNames(env, deployment) {
+    const own = `mason_${deployment.replace(/[^A-Za-z0-9_]/g, "_")}`;
+    const publication = env.PG_PUBLICATION || own;
+    return { publication, slot: env.PG_SLOT_NAME || own, owned: publication === own };
+}
+/** The connection options from the `PG_*` environment (DATABASE_URL already laid under it). */
+export function pgOptions(env, replication = false) {
+    const need = (k) => {
+        const v = env[k];
+        if (!v)
+            throw new Error(`${k} is not set (DATABASE_URL, or the PG_* variables)`);
+        return v;
+    };
+    const port = parsePort("PG_PORT", env.PG_PORT, 5432);
+    if (port.problem !== undefined)
+        throw new Error(port.problem);
+    return {
+        host: need("PG_HOST"), port: port.port, database: need("PG_DATABASE"), user: need("PG_USER"),
+        password: env.PG_PASSWORD ?? "", sslmode: (env.PG_SSLMODE || "prefer"),
+        replication, applicationName: replication ? "mason-cdc" : "mason",
+    };
+}
+const NO_SLOT = { exists: false, active: false, activePid: null, plugin: null, database: null, confirmedFlush: null,
+    walStatus: null, retainedBytes: null, safeBytes: null };
+export async function readSlot(pg, slot) {
+    const r = await pg.query(`SELECT plugin, database, active, active_pid::text, confirmed_flush_lsn::text, wal_status, safe_wal_size::text, ` +
+        `pg_wal_lsn_diff((CASE WHEN pg_is_in_recovery() THEN pg_last_wal_replay_lsn() ELSE pg_current_wal_lsn() END), restart_lsn)::text ` +
+        `FROM pg_replication_slots WHERE slot_name = ${pgLiteral(slot)}`);
+    const row = r.rows[0];
+    if (!row)
+        return NO_SLOT;
+    const big = (v) => (v === null || v === undefined || v === "" ? null : BigInt(v.split(".")[0]));
+    return { exists: true, plugin: row[0], database: row[1], active: row[2] === "t", activePid: row[3] ?? null, confirmedFlush: row[4] ?? null,
+        walStatus: row[5] ?? null, safeBytes: big(row[6]), retainedBytes: big(row[7]) };
+}
+/** Drop the slot (it must not be active). */
+export async function dropSlot(pg, slot) {
+    await pg.query(`SELECT pg_drop_replication_slot(${pgLiteral(slot)})`);
+}
+/** Where the source's WAL is now: the insert position on a primary, the replayed position on a standby (a
+ *  logical slot on a standby decodes only what it has replayed). */
+export async function currentLsn(pg) {
+    const r = await pg.query("SELECT (CASE WHEN pg_is_in_recovery() THEN pg_last_wal_replay_lsn() ELSE pg_current_wal_lsn() END)::text");
+    return parseLsn(r.rows[0]?.[0] ?? "");
+}
+/** Create the slot over a replication connection; returns the WAL position it starts at. */
+export async function createSlot(repl, slot) {
+    // NOEXPORT_SNAPSHOT: the first copy reads each table by key pages, each its own statement (the stream's
+    // versions win wherever they overlap) — no long snapshot to hold open.
+    const r = await repl.query(`CREATE_REPLICATION_SLOT ${pgIdent(slot)} LOGICAL pgoutput NOEXPORT_SNAPSHOT`);
+    return r.rows[0]?.[1] ?? "";
+}

package/dist/src/cli/background.js

@@ -0,0 +1,174 @@
+/**
+ * `mason start` IN THE BACKGROUND (an interface as good for people as for coding agents).
+ * The run is a process of its own (`mason start … --attach`, detached, its terminal the file `dev.log` in the run's
+ * folder): this shows the run's start as it happens, and returns once the run says it is ready — or with the run's own
+ * exit code when it stops first (a wrong password), its reason already shown. A coding agent's shell can
+ * then go on (`mason tables`, `mason add`, `mason wait`), and a person's terminal is theirs again.
+ *
+ * A second `mason start` for a project whose run is up does not start another: it says so and returns 0, so running it
+ * again is always safe.
+ *
+ * What is detached is the run's SUPERVISOR (src/cli/supervise.ts), which starts the run and starts it again if it dies
+ * once it was ready; a start that fails ends the supervisor with the run's own exit code, so this reads it as before.
+ */
+import { choosePgPort } from "./ports.js";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { spawn } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { runnable } from "../lib/runnable.js";
+import { READY, STEP_MARK } from "./dev.js";
+import { liveRun } from "./runs.js";
+import { theProject } from "./project.js";
+import { runListenerSettings } from "./pg-tls.js";
+import { endText, failed, jsonText, nextBlock } from "./command.js";
+import { DEV } from "./registry.js";
+import { out as styleOut, stepRenderer } from "./style.js";
+import { postgresLink, withoutPlaceholders } from "./pg-url.js";
+const say = (m) => { process.stdout.write(`${m}\n`); };
+const MAIN = path.join(path.dirname(fileURLToPath(import.meta.url)), "main.ts");
+const SUPERVISE = path.join(path.dirname(fileURLToPath(import.meta.url)), "supervise.ts");
+/** The commands to run next, as `mason start` prints them once the run is up: one a line, what each does beside it
+ *  (not `mason status | mason logs | mason stop` on one line). */
+export const NEXT_STEPS = [
+    ["mason tables", "the database's tables, and what you can add"],
+    ["mason add <table>", "copy a table into Mason"],
+    ["mason user add <name>", "a user for a BI tool or your app: its password and URL, printed once"],
+    ["mason status", "what Mason is doing"],
+    ["mason logs", "its log: every copy, poll and statement"],
+    ["mason stop", "stop Mason; the next mason start resumes where it stopped"],
+];
+/** `args` are `mason start`'s own, passed on to the run with --attach. */
+export async function devInBackground(args, o) {
+    const t = theProject(process.cwd());
+    if ("problem" in t)
+        return failed("start", t, { json: o.json });
+    const home = t.project.dir;
+    const other = await liveRun(t.project);
+    if (other !== null) {
+        // running already is what start wants: said, exit 0, the same answer as a start (what is up, and what to run)
+        if (o.json) {
+            say(jsonText(DEV, { running: true, pid: other.pid, postgres: postgresLink(other.postgresUrl) }, [["mason status", "what it is doing"], ["mason stop", "stop it"]]));
+            return 0;
+        }
+        say(`Mason is already running for ${t.project.config.name} (pid ${other.pid}).`);
+        say(nextBlock([["mason status", "what it is doing"], ["mason stop", "stop it"]]));
+        return 0;
+    }
+    // a MASON_PG_PORT that is no port, or is held, is refused HERE, where the person (or --json) reads it, before a run is
+    // spawned; the run checks again for itself (choosePgPort, src/cli/ports.ts), since a port may be taken in between
+    const pg = await choosePgPort(process.env.MASON_PG_PORT, new Set(), t.project.config.pgPort ?? null);
+    if ("problem" in pg)
+        return failed("start", pg, { json: o.json });
+    // and so is a listener off loopback without TLS, by the run's own check (src/cli/pg-tls.ts): read, never written
+    const listen = runListenerSettings(process.env, { port: pg.port, selfSigned: o.selfSignedTls, home });
+    if ("problem" in listen)
+        return failed("start", listen, { json: o.json });
+    fs.mkdirSync(home, { recursive: true, mode: 0o700 });
+    // the run's terminal: its setup URL carries the run's token and it prints the API key, so only the developer reads it
+    const logFile = path.join(home, "dev.log");
+    const fd = fs.openSync(logFile, "a", 0o600);
+    fs.chmodSync(logFile, 0o600);
+    const from = fs.fstatSync(fd).size;
+    fs.writeSync(fd, `\n--- mason start ${new Date().toISOString()} ---\n`);
+    // the run is started IN the project's root, so it finds the same project
+    const run = [process.execPath, runnable(MAIN), "start", ...args.filter((a) => a !== "--json"), "--attach"];
+    const child = spawn(process.execPath, [runnable(SUPERVISE), home, logFile, ...run], { detached: true, stdio: ["ignore", fd, fd], cwd: t.project.root, env: process.env });
+    fs.closeSync(fd);
+    let exited = null;
+    child.on("exit", (code, signal) => { exited = code ?? (signal ? 1 : 0); });
+    // Ctrl-C while it starts stops the start too: nothing is left running that nobody asked for
+    const abort = () => { child.kill("SIGTERM"); };
+    process.on("SIGINT", abort);
+    // what it does, as a checklist (as `mason init` draws its own): the run's steps, drawn as they happen; under
+    // --json nothing until it is up (one document), the lines after the last step kept as a failure's why
+    const st = styleOut();
+    const line = (x) => { if (!o.json)
+        say(x); };
+    line(st.bold(`Starting Mason for ${t.project.config.name}`));
+    const steps = stepRenderer({ line, live: !o.json && process.stdout.isTTY ? { write: (x) => { process.stdout.write(x); } } : null, style: st });
+    let begun = false;
+    let offset = from;
+    let partial = "";
+    let postgres = null;
+    const since = []; // the run's lines after its last step: a failure's reasons
+    try {
+        for (;;) {
+            const size = fs.statSync(logFile).size;
+            if (size > offset) {
+                const buf = Buffer.alloc(size - offset);
+                const r = fs.openSync(logFile, "r");
+                try {
+                    fs.readSync(r, buf, 0, buf.length, offset);
+                }
+                finally {
+                    fs.closeSync(r);
+                }
+                offset = size;
+                const lines = (partial + buf.toString()).split("\n");
+                partial = lines.pop() ?? "";
+                for (const l of lines) {
+                    if (l.startsWith("--- mason start "))
+                        continue;
+                    if (l === "" && steps.open() === null && !begun)
+                        continue; // the log's spacing before the first step
+                    begun = true;
+                    if (l === READY) {
+                        child.unref();
+                        const next = NEXT_STEPS;
+                        // the run's pid, as it recorded it (the child is its supervisor), the one mason stop signals
+                        const pid = JSON.parse(fs.readFileSync(path.join(home, "config.json"), "utf8")).running?.pid ?? null;
+                        if (o.json)
+                            say(jsonText(DEV, { running: true, pid, postgres }, next));
+                        else
+                            say(endText({ next }, st));
+                        return 0;
+                    }
+                    // the run's steps (src/cli/dev.ts STEP_MARK) as a checklist; any other line as it came
+                    if (l.startsWith(STEP_MARK.begin)) {
+                        since.length = 0;
+                        steps.begin(l.slice(STEP_MARK.begin.length));
+                        continue;
+                    }
+                    if (l.startsWith(STEP_MARK.done)) {
+                        since.length = 0;
+                        steps.done(l.slice(STEP_MARK.done.length));
+                        continue;
+                    }
+                    if (l.startsWith(STEP_MARK.fail)) {
+                        since.length = 0;
+                        steps.fail(l.slice(STEP_MARK.fail.length));
+                        continue;
+                    }
+                    // the run's line (src/cli/dev.ts: "Postgres  <url>")
+                    const pgLine = /^\s+Postgres\s+(postgresql:\S+)/.exec(l);
+                    if (pgLine) {
+                        postgres = postgresLink(withoutPlaceholders(pgLine[1]));
+                        steps.print(`  ${st.dim("Postgres API")}  ${pgLine[1]}  ${st.dim("(mason user add <name> makes a user)")}`);
+                        continue;
+                    }
+                    if (/^\s+Web UI\s+mason ui\b/.test(l)) {
+                        steps.print(`  ${st.dim("Web UI".padEnd("Postgres API".length))}  ${st.cyan("mason ui")}  ${st.dim("(the status page and the users, in your browser)")}`);
+                        continue;
+                    }
+                    if (l.trim())
+                        since.push(l.trim().replace(/^mason start: /, ""));
+                    steps.print(l);
+                }
+            }
+            if (exited !== null) {
+                if (steps.open() !== null)
+                    steps.fail();
+                if (partial)
+                    since.push(partial.trim());
+                // the run's own last words are the reason (a wrong password), already shown line by line to a person
+                return failed("start", { message: `the run stopped before it was ready (exit ${exited})`, code: "failed", why: o.json && since.length ? since.join("\n") : undefined,
+                    fix: `the whole log: ${logFile}`, exit: exited === 0 ? 1 : exited }, { json: o.json });
+            }
+            await new Promise((r) => setTimeout(r, 150));
+        }
+    }
+    finally {
+        process.off("SIGINT", abort);
+    }
+}

package/dist/src/cli/command.js

@@ -0,0 +1,172 @@
+/**
+ * A `mason` COMMAND, DESCRIBED ONCE (every command, present and future, is and stays self-documenting, and that is
+ * checked). Each command is a `CommandSpec`: its name, the one line `mason --help` lists it with, its usage
+ * (what `mason <name> --help` prints), and the options its parser accepts — the SAME object the parser reads, so an
+ * option cannot exist without its spec. src/cli/main.ts builds the command table and the top-level help from the
+ * specs, and tests/cli-help.test.ts holds every one to the rules below, a command added later included:
+ *
+ *   - `mason <name> --help` and `mason help <name>` print its usage and exit 0;
+ *   - the usage starts `usage: mason <name>`, and shows at least one example (an indented `mason <name> …` line);
+ *   - every option the parser takes is in the usage, by its `--name` (and `-x` when it has one);
+ *   - `mason --help` lists it with its summary;
+ *   - a wrong option prints what is wrong, then the usage, and exits 2.
+ */
+import { parseArgs } from "node:util";
+import { err as styleErr, out as styleOut, PLAIN, wrap } from "./style.js";
+const HELP = { help: { type: "boolean", short: "h" } };
+/**
+ * A command line read by its spec: the values and positionals; or the exit code when it answered itself — 0 after
+ * printing the usage for `--help`, 2 after saying what is wrong with the line, then the usage.
+ */
+export function parseCommand(spec, argv) {
+    let parsed;
+    try {
+        parsed = parseArgs({ args: argv, options: { ...spec.options, ...HELP }, allowPositionals: true });
+    }
+    catch (e) {
+        return failed(spec.name, { message: e.message, code: "usage", usage: spec.usage, next: [[`mason ${spec.name} --help`, "its usage, with every option"]] }, { json: wantsJson(argv) });
+    }
+    if (parsed.values.help) {
+        process.stdout.write(spec.usage);
+        return 0;
+    }
+    return { values: parsed.values, positionals: parsed.positionals };
+}
+/** The column a step's `what` starts in, after the two-space indent: wide enough for `mason add orders --columns …`. */
+const NEXT_WIDTH = 44;
+/**
+ * WHAT TO RUN NEXT, in one shape for every command (some commands showed Next as one line, some without the Next
+ * prefix and some as a kind of table; the table won): a blank line, `Next:`, then each command with what
+ * it does in a column, after everything else a command prints — a failure's way out included (failureText). With --json the
+ * same steps are the answer's `next` (nextJson). tests/conventions.test.ts refuses a `Next` or `Try` written any other way.
+ */
+export function nextBlock(steps, s = styleOut()) {
+    // on a terminal: `Next:` bold, each command cyan, what it does dim and wrapped under its own column (src/cli/style.ts)
+    const step = ([command, what]) => {
+        if (!what)
+            return `  ${s.cyan(command)}`;
+        const used = 2 + Math.max(command.length, NEXT_WIDTH) + 2;
+        return `  ${s.cyan(command.padEnd(NEXT_WIDTH))}  ${s.dim(wrap(what, s.width, 2 + NEXT_WIDTH + 2, used))}`;
+    };
+    return ["", s.bold("Next:"), ...steps.map(step)].join("\n");
+}
+export function reportText(r, s = styleOut()) {
+    const lines = [s.bold(r.headline)];
+    const w = Math.max(0, ...(r.details ?? []).map(([label]) => label.length));
+    for (const [label, value] of r.details ?? [])
+        lines.push(`  ${s.dim(label.padEnd(w))}   ${wrap(value, s.width, 2 + w + 3)}`);
+    return withEnd(lines.join("\n"), r, s);
+}
+/**
+ * The end of a command's output, to print on its own (out(endText(…))): its tips, then what to run next, each part after
+ * a blank line; "" when it has neither. withEnd puts it after text a command has not printed yet.
+ */
+export function endText(r, s = styleOut()) {
+    const tips = r.tips?.length ? `\n${r.tips.map((t) => s.dim(`Tip: ${wrap(t, s.width, 5)}`)).join("\n")}` : "";
+    const next = r.next?.length ? nextBlock(r.next, s) : "";
+    return tips && next ? `${tips}\n${next}` : tips || next;
+}
+export function withEnd(text, r, s = styleOut()) {
+    const end = endText(r, s);
+    return end ? `${text}\n${end}` : text;
+}
+export const STDIO = { out: (m) => { process.stdout.write(`${m}\n`); }, err: (m) => { process.stderr.write(`${m}\n`); } };
+/** The steps as a program reads them: `[{ command, what }]`, the same list the human output prints under Next. */
+export function nextJson(steps = []) {
+    return steps.map(([command, what]) => what ? { command, what } : { command });
+}
+/** Whether a command line asked for data: `--json` anywhere on it (read before the line is parsed, so a wrong line
+ *  is answered in JSON too). */
+export const wantsJson = (argv) => argv.includes("--json");
+/**
+ * A COMMAND'S --json ANSWER (an agent wants clear, unambiguous context and a machine-readable format, and a person
+ * and an agent both want the next steps): one JSON object on stdout, always with `next` — the same steps the human
+ * output prints — and only the keys the command's spec declares (`CommandSpec.json`), so the help's shape and the answer
+ * cannot drift: a key the spec does not name is a bug here, said as one. A list is wrapped in an object for the same
+ * reason (`{ tables: […], next }`), never printed bare.
+ */
+export function jsonText(spec, value, next = []) {
+    const declared = spec.json;
+    if (declared === undefined)
+        throw new Error(`mason ${spec.name} prints JSON, and its spec declares no json keys (src/cli/command.ts CommandSpec.json)`);
+    const stray = Object.keys(value).filter((k) => !declared.includes(k));
+    if (stray.length)
+        throw new Error(`mason ${spec.name} --json: ${stray.join(", ")} ${stray.length === 1 ? "is" : "are"} not in its spec's json keys (${declared.join(", ")}): add ${stray.length === 1 ? "it" : "them"} there and to its usage`);
+    if (!declared.includes("next"))
+        throw new Error(`mason ${spec.name}: its json keys must end with next (every --json answer carries what to run next)`);
+    return JSON.stringify({ ...value, next: nextJson(next) }, null, 2);
+}
+/**
+ * THE CODES A FAILURE CARRIES, a closed set a program can branch on (`mason help agent` lists them).
+ */
+export const FAILURE_CODES = {
+    usage: "the command line is wrong: a flag, an argument, a value (exit 2; the usage follows for a person)",
+    no_project: "no Mason project here or above (mason init makes one)",
+    bad_config: "this folder is a Mason project whose .mason/config.json cannot be read: repair it, or move .mason/ aside and mason init",
+    bad_source: "the project's sources file (mason/sources/<name>.yml) is missing or names something wrong: the file, the key and the way out are said",
+    not_running: "Mason is not running for the project (mason start)",
+    unreachable: "Mason is running but does not answer (mason logs says why)",
+    not_found: "a name nothing has: a table, a column, a user",
+    nothing_yet: "nothing to work on yet: no table added (the next steps say what adds one)",
+    refused: "Mason refused what was asked, with the reason",
+    failed: "the command tried and could not finish: the reason, and what to do",
+    timeout: "ran out of time — mason wait; nothing stopped (exit 124)",
+};
+/** A failure thrown from deep in a command, for main.ts to print: the one way a failure leaves a command without a return code. */
+export class CommandFailed extends Error {
+    failure;
+    constructor(f) { const n = asFailure(f); super(n.message); this.failure = n; }
+}
+export const asFailure = (f) => "problem" in f ? { ...f, message: f.problem } : f;
+/** The failure as a person reads it (above): a red ✘ (wherever a step fails, as a green tick shows wherever one
+ *  succeeds — the checklist's own mark, ✗ where nothing is styled), the command's name and the
+ *  message in bold, then why and fix as rows, then Next, then a wrong line's usage. */
+export function failureText(cmd, f, s = styleErr()) {
+    const head = cmd ? `mason ${cmd}:` : "mason:";
+    // the checklist's marks: ✘ on a terminal (beside its ✔), ✗ in plain text (beside its ✓); a style that colours is one
+    const mark = s.red("x") === "x" ? "✗" : "✘";
+    const lines = [`${s.red(mark)} ${s.red(head)} ${s.bold(wrap(f.message, s.width, 2, mark.length + 1 + head.length + 1))}`];
+    const rows = [...(f.why ? [["why", f.why]] : []), ...(f.fix ? [["fix", f.fix]] : [])];
+    // a value of several lines (a list of failures, an entry to paste) keeps its lines, each under the first
+    for (const [label, value] of rows)
+        lines.push(`  ${s.dim(label)}   ${value.split("\n").map((l, i) => i ? `${" ".repeat(8)}${wrap(l, s.width, 8)}` : wrap(l, s.width, 8)).join("\n")}`);
+    let text = lines.join("\n");
+    if (f.next?.length)
+        text += `\n${nextBlock(f.next, s)}`;
+    if (f.usage)
+        text += `\n\n${f.usage.replace(/\n$/, "")}`;
+    return text;
+}
+/** A failure's `error` as --json carries it: on its own (failureJson), or beside a command's own keys when it did part of
+ *  its work (mason add: the tables added; mason wait: the tables' state). */
+export function errorJson(f) {
+    return { code: f.code, message: f.message, ...(f.why ? { why: f.why } : {}), ...(f.fix ? { fix: f.fix } : {}) };
+}
+/** The failure as a program reads it: `{"error": {code, message, why?, fix?}, "next": […]}`. */
+export function failureJson(f) {
+    return JSON.stringify({ error: errorJson(f), next: nextJson(f.next) }, null, 2);
+}
+/**
+ * SAY A FAILURE AND RETURN ITS EXIT CODE — the one way a command reports that it could not do what was asked: for a
+ * person on stderr (failureText), under --json as one JSON document on stdout (failureJson), so a program that asked for
+ * data never has to read prose. tests/conventions.test.ts refuses a `mason <cmd>: …` written to stderr any other way.
+ */
+export function failed(cmd, f, o) {
+    const n = asFailure(f);
+    const io = o.io ?? STDIO;
+    if (o.json)
+        io.out(failureJson(n));
+    else
+        io.err(failureText(cmd, n, o.style ?? failureStyle(io)));
+    return exitOf(n);
+}
+/** The style a failure is drawn in: stderr's own when it goes to stderr, plain for an io a test or a caller supplied.
+ *  By the STREAM, never by "was an io passed" (CLAUDE.md "Two readers, one interface"): mason init defaults its io to
+ *  STDIO and passed it on, and every failure it drew was plain — no red head, no dim labels — on a terminal. */
+export const failureStyle = (io) => io === STDIO ? styleErr() : PLAIN;
+/** A failure's exit code: its code's (2 a wrong command line; 124 anything that ran out of time, a wait or a query Mason
+ *  did not answer in time; else 1), unless it names one. The one place it is decided, so the same refusal exits the
+ *  same wherever it is found. */
+export function exitOf(f) {
+    return f.exit ?? (f.code === "usage" ? 2 : f.code === "timeout" ? 124 : 1);
+}