@mason-data/cli 0.0.0-stage → 0.6.2

package/dist/src/cli/doctor.js

@@ -0,0 +1,257 @@
+/**
+ * `mason doctor` (an agent's report on #409 asked for a one-shot check of what the machine lacks): what this machine
+ * has of what mason start needs — Node, memory, the ports — and, when a run is up or a database is named, the
+ * database and the run's health. Every failing check says what to do; the report ends with the next command to run.
+ * Read-only: nothing is started, written or changed.
+ */
+import { failed, jsonText, nextBlock, parseCommand } from "./command.js";
+import { out as styleOut, wrap } from "./style.js";
+import { heldPgPort, namedPgPort, portHolder } from "./ports.js";
+import { callRun, RunUnreachable } from "./control.js";
+import { readiness } from "./commands.js";
+import { connectSource, maskedUrl, parseSource, readSourceFacts, versionProblem } from "./preflight.js";
+import { theRun } from "./runs.js";
+import { projectSource } from "./project.js";
+import { DEFAULT_PG_PORT } from "../api/postgres/listener.js";
+import { defaultMemoryLimit, machineMemoryBytes, MEMORY_LIMIT_RE } from "../duck/store.js";
+const out = (m) => { process.stdout.write(`${m}\n`); };
+/**
+ * THE NODE THE PACKAGE NEEDS, major.minor: cli/package.json's `engines`, this check, `mason --help` and the README read
+ * or are held to it (tests/cli-doctor.test.ts). The package is compiled to JavaScript (cli/build.ts), so it needs no
+ * type stripping; 22.13 is the floor the published package was measured against (22.12 refuses a module it
+ * loads, 22.23 loads every module). A checkout and the image run the TypeScript itself and need 23.6 (the root
+ * package.json).
+ */
+export const NODE_MIN = "22.13";
+export const DOCTOR_USAGE = `usage: mason doctor [--json]
+
+  Says whether this machine is ready for Mason — Node ${NODE_MIN} or later, memory (Mason's copy uses up to half of it by
+  default; MASON_DUCKDB_MEMORY=4GiB sets it), the ports Mason listens on — with the way out for anything missing. In a
+  project's folder it checks the project's database too; with Mason running for it, the run in a few lines instead: its
+  database, whether every added table is copied, the statements of the last day it refused or answered slowly.
+
+    mason doctor
+
+  --json   as data: { ready, verdict, checks: [{ name, ok, detail }], next: [{ command, what }] }; ok is true, false,
+           or null when it could not tell
+
+  Exit status: 0 everything checked is fine; 1 something is not (said, with the way out).
+`;
+export const DOCTOR = {
+    name: "doctor",
+    summary: "check this machine for what mason start needs (Node, memory, ports, the database, the run) and say what to do",
+    usage: DOCTOR_USAGE,
+    options: { json: { type: "boolean" } },
+    json: ["ready", "verdict", "checks", "next"],
+};
+/** The ports a mason start takes by default, and what listens on each. */
+export const PORTS = [[4039, "the mason commands"], [DEFAULT_PG_PORT, "the Postgres API"]];
+/** Node's version against the package's floor, major then minor. */
+export function nodeCheck(version = process.versions.node) {
+    const [major, minor] = version.split(".").map(Number);
+    const [minMajor, minMinor] = NODE_MIN.split(".").map(Number);
+    return major > minMajor || (major === minMajor && minor >= minMinor)
+        ? { name: "node", ok: true, detail: `Node ${version}` }
+        : { name: "node", ok: false, detail: `Node ${version} is too old: mason needs ${NODE_MIN} or later (nodejs.org, or nvm install ${minMajor})` };
+}
+/** The machine's memory and what the copy's engine may use of it (src/duck/store.ts DEFAULT_MEMORY_LIMIT, MASON_DUCKDB_MEMORY). */
+export function memoryCheck(totalBytes = machineMemoryBytes(), env = process.env) {
+    const gib = (totalBytes / 1024 ** 3).toFixed(1);
+    const set = env.MASON_DUCKDB_MEMORY;
+    if (set !== undefined && !MEMORY_LIMIT_RE.test(set)) {
+        return { name: "memory", ok: false, detail: `MASON_DUCKDB_MEMORY=${JSON.stringify(set)} is not a size: write it as 4GiB, 512MiB or 2GB (mason start refuses it too)` };
+    }
+    const share = set ?? defaultMemoryLimit(totalBytes);
+    const low = totalBytes < 2 * 1024 ** 3;
+    return {
+        name: "memory", ok: !low,
+        detail: `${gib} GiB; Mason's copy uses up to ${share.trim()}${set !== undefined ? " (MASON_DUCKDB_MEMORY)" : " (half, the default; MASON_DUCKDB_MEMORY sets it)"}` +
+            (low ? ". Under 2 GiB, a large table's copy and a query starve each other: expect statements refused for memory" : ""),
+    };
+}
+/**
+ * Mason's ports in one line, decided by the probe mason start uses (`portHolder`, src/cli/ports.ts: every address, so
+ * doctor and start agree on a port another program holds on the wildcard). A taken command port only moves Mason to
+ * another free one, so the line says that and who holds each (this project's Mason, or another program). The Postgres
+ * API's port is the project's (`recorded`, src/cli/ports.ts choosePgPort) or the one the developer NAMED (`MASON_PG_PORT`),
+ * and is not moved: held by anything but this project's run, it is the failure mason start will give.
+ * The ports line used to confuse: it was a "?" naming each port's role and "something else".
+ */
+export async function portsCheck(runs, taken = portHolder, env = process.env, recorded = null) {
+    const named = env.MASON_PG_PORT ? namedPgPort(env.MASON_PG_PORT) : null;
+    if (named && "problem" in named)
+        return { name: "ports", ok: false, detail: `${named.problem}: ${named.fix}` };
+    const pgPort = named ? named.port : recorded ?? DEFAULT_PG_PORT;
+    const ports = PORTS.map(([p]) => p === DEFAULT_PG_PORT ? pgPort : p);
+    const byHolder = new Map();
+    for (const port of ports) {
+        const held = await taken(port);
+        if (!held)
+            continue;
+        const holder = runs.find((r) => [r.control.url, r.postgresUrl].some((u) => portOf(u) === port));
+        if (port === pgPort && !holder) {
+            const refused = heldPgPort(port, held);
+            return { name: "ports", ok: false, detail: `${refused.problem}: ${refused.why}. ${refused.fix}` };
+        }
+        const who = holder ? `Mason for ${holder.name}` : "another program";
+        byHolder.set(who, [...(byHolder.get(who) ?? []), port]);
+    }
+    if (!byHolder.size)
+        return { name: "ports", ok: true, detail: `${ports.join(", ")} free` };
+    const used = [...byHolder].map(([who, ps]) => `${ps.join(", ")} in use by ${who}`).join("; ");
+    return { name: "ports", ok: true, detail: `${used}: mason start will use another free port for the commands (mason status shows them)` };
+}
+function portOf(url) {
+    try {
+        const u = new URL(url);
+        return u.port ? Number(u.port) : null;
+    }
+    catch {
+        return null;
+    } // not a URL: no port to compare with
+}
+/** The database a URL names: reachable, its version, what this user can read. */
+export async function databaseCheck(url) {
+    const parsed = parseSource(url);
+    if ("problem" in parsed)
+        return { name: "database", ok: false, detail: parsed.problem };
+    const connected = await connectSource(parsed.source);
+    if ("failure" in connected)
+        return { name: "database", ok: false, detail: `${maskedUrl(parsed.source.url)}: ${connected.failure.message}${connected.failure.hints.map((h) => `\n    - ${h}`).join("")}` };
+    let facts;
+    try {
+        facts = await readSourceFacts(connected.pg);
+    }
+    finally {
+        connected.pg.close();
+    }
+    const tooOld = versionProblem(facts);
+    if (tooOld)
+        return { name: "database", ok: false, detail: `${maskedUrl(parsed.source.url)}: ${tooOld}` };
+    return { name: "database", ok: true, detail: `${maskedUrl(parsed.source.url)}: Postgres ${facts.versionText}${facts.standby ? " (a read replica)" : ""}, ${facts.tables.length} table(s) this user can read` };
+}
+/** The run's health, from its own state: every table copied, failing, or still at work; and what it refused or answered slowly. */
+export async function runChecks(run) {
+    const checks = [];
+    let state;
+    try {
+        const r = await callRun(run, "GET", "/api/state");
+        if (r.status !== 200)
+            throw new RunUnreachable(String(r.body.error ?? `HTTP ${r.status}`));
+        state = r.body;
+    }
+    catch (e) {
+        if (!(e instanceof RunUnreachable))
+            throw e;
+        return [{ name: "mason start", ok: false, detail: `${run.name}: ${e.message}` }];
+    }
+    checks.push({ name: "mason start", ok: true, detail: `${run.name} (pid ${run.pid}); Postgres ${state.source.version}${state.source.standby ? " (a read replica)" : ""} at ${state.source.url}` });
+    const ready = readiness(state);
+    const picked = state.tables.filter((t) => t.added).length;
+    if (ready.failed.length)
+        checks.push({ name: "project", ok: false, detail: `${ready.failed.length} problem(s): ${ready.failed.join("; ")} — mason wait prints each with the way out` });
+    else if (ready.waiting.length)
+        checks.push({ name: "project", ok: null, detail: `still at work: ${ready.waiting.join("; ")} (mason wait)` });
+    else
+        checks.push({ name: "project", ok: true, detail: picked ? `${picked} table(s) copied and kept current` : "no table added yet (mason tables, then mason add <table>)" });
+    const slow = await callRun(run, "GET", "/api/slow"), refused = await callRun(run, "GET", "/api/refused");
+    const slowN = slow.status === 200 ? slow.body.slow.length : null, refusedN = refused.status === 200 ? refused.body.refused.length : null;
+    checks.push(slowN === null || refusedN === null
+        ? { name: "queries", ok: null, detail: "the last day's statements could not be read (mason status)" }
+        : { name: "queries", ok: slowN + refusedN === 0, detail: slowN + refusedN === 0 ? "none refused or slow in the last day" : `last day: ${refusedN} refused, ${slowN} slow (mason status lists each, with what to do)` });
+    return checks;
+}
+/** What to run next, from what the checks found. */
+export function nextSteps(checks, run, project) {
+    const bad = (name) => checks.some((c) => c.name === name && c.ok === false);
+    if (bad("node"))
+        return [["mason doctor", `again, once Node ${NODE_MIN} or later is installed`]];
+    if (bad("project") && run === null)
+        return [["mason doctor", "again, once .mason/config.json is repaired or moved aside (the detail above says how)"]];
+    if (bad("database"))
+        return [["mason start", `once the connection works (the detail above says how; the URL is in ${project ? `${project.dir}/config.json` : ".mason/config.json"})`]];
+    if (project === null)
+        return [
+            ["mason init", "in your app's folder: the database from DATABASE_URL in .env.local, .env or the shell"],
+            ["mason init postgres://user:password@localhost:5432/your_app", "or name it"],
+        ];
+    if (run === null)
+        return [["mason start", "Mason in the background, for this folder"]];
+    if (bad("mason start"))
+        return [["mason logs", "why Mason does not answer"], ["mason stop", "then mason start, to start it again"]];
+    if (bad("project"))
+        return [["mason wait", "each problem, with the way out"]];
+    if (checks.some((c) => c.name === "project" && /no table added/.test(c.detail)))
+        return [["mason tables", "the database's tables and their columns"], ["mason add <table>", "copy a table into Mason"]];
+    if (bad("queries"))
+        return [["mason status", "the refused and slow statements, each with what to do"]];
+    return [["mason user add <name>", "a user for a BI tool or your app: its password and URL, printed once"]];
+}
+/** The one line a first run reads: ready for the next step, or what is missing. */
+export function verdictOf(checks, run, project) {
+    const failed = checks.filter((c) => c.ok === false);
+    if (run !== null) {
+        return failed.length
+            ? { ready: false, verdict: `Mason is running for ${run.name}, with ${failed.length} problem${failed.length === 1 ? "" : "s"}` }
+            : { ready: true, verdict: `Mason is running for ${run.name}` };
+    }
+    // the first failure's headline: its sentence before the parenthesis or the way out, which the line below carries whole
+    if (failed.length)
+        return { ready: false, verdict: `Not ready: ${failed[0].detail.split(/ \(|\. /)[0].trim()}` };
+    return { ready: true, verdict: project ? `Ready: mason start (the project ${project.config.name})` : "Ready: mason init, in your app's folder" };
+}
+/** A check's mark: ok, !! a failure, ? could not tell. */
+function markOf(c, s) {
+    return c.ok === false ? s.red("!! ") : c.ok === null ? s.yellow("?  ") : s.green("ok ");
+}
+export async function doctor(argv, deps = {}) {
+    const f = parseCommand(DOCTOR, argv);
+    if (typeof f === "number")
+        return f;
+    if (f.positionals.length)
+        return failed("doctor", { message: "it takes no URL", code: "usage", fix: "in a project's folder it checks the project's database (mason init makes one)", usage: DOCTOR_USAGE }, { json: f.values.json === true });
+    const which = await (deps.which ?? theRun)(process.cwd());
+    const run = "run" in which ? which.run : null;
+    const project = which.project;
+    // THE CHECKS FOR THE MOMENT (the first run's output, every check at once, confused): before a start, whether
+    // the machine can run one; with a run up, the run itself — its ports are its own, and the machine is already proven
+    const checks = [];
+    if (run === null) {
+        checks.push(nodeCheck());
+        checks.push(memoryCheck());
+        checks.push(await portsCheck([], portHolder, process.env, project?.config.pgPort ?? null));
+        if (project) {
+            // the database the project's sources file names (src/cli/sources.ts): a file that cannot be read is the failing line
+            const src = projectSource(project);
+            checks.push("message" in src ? { name: "database", ok: false, detail: `${src.message}${src.why ? `: ${src.why.replace(/\n/g, "; ")}` : ""}. ${src.fix?.split("\n")[0] ?? ""}`.trim() }
+                : await databaseCheck(src.resolved));
+        }
+        // a project here whose config cannot be read: init would refuse, so "Ready: mason init" would send a person nowhere
+        if ("code" in which && which.code === "bad_config")
+            checks.push({ name: "project", ok: false, detail: `${which.problem}. ${(which.fix ?? "").replace(/^./, (c) => c.toUpperCase())}`.trim() });
+    }
+    else {
+        checks.push(...await runChecks(run));
+    }
+    const next = nextSteps(checks, run, project);
+    const { ready, verdict } = verdictOf(checks, run, project);
+    if (f.values.json) {
+        out(jsonText(DOCTOR, { ready, verdict, checks }, next));
+        return ready ? 0 : 1;
+    }
+    // on a terminal: the verdict green or red, each check's mark by its state, its detail wrapped under its column
+    const s = styleOut();
+    out(s.bold(ready ? s.green(verdict) : s.red(verdict)));
+    const w = Math.max(...checks.map((c) => c.name.length));
+    for (const c of checks) {
+        if (run !== null && c.name === "mason start" && c.ok === true)
+            continue; // the verdict line said it
+        const mark = markOf(c, s);
+        out(`  ${mark} ${c.name.padEnd(w)}  ${wrap(c.detail, s.width, 2 + 3 + 1 + w + 2)}`);
+    }
+    if (run !== null)
+        out(s.dim(`  its copy of the data and its state in ${run.home}. mason status for the details`));
+    if (run === null || !ready)
+        out(nextBlock(next));
+    return ready ? 0 : 1;
+}

package/dist/src/cli/guides.js

@@ -0,0 +1,134 @@
+/**
+ * THE GUIDES `mason help` PRINTS (once `mason docs`; under `mason help` because help is the more common word): OFFLINE,
+ * from the CLI itself, short enough to read whole. An agent using 0.1.7 found what the CLI accepts by grepping the
+ * package's compiled code, because the help pointed at docs the package does not ship. So the reference is here, and
+ * every list in it is the code's own table (the commands from the registry, the failure codes from src/cli/command.ts):
+ * a guide cannot list a command or a code the CLI does not have.
+ *
+ * A guide's name is never a command's (tests/cli-help.test.ts): one word never means two pages.
+ */
+/** The guides, by the name `mason help <guide>` takes. */
+export const GUIDES = {
+    agent: "driving Mason from a coding agent: the commands in order, --json and its shapes, exit codes, failures",
+    production: "going to production: the mason-data image, its settings, Docker Compose, logical replication, upgrades",
+};
+/** A guide's text: built at run time from the registry's and the commands' own tables. */
+export async function guideText(guide) {
+    if (guide === "agent")
+        return agentGuide();
+    return productionGuide();
+}
+/**
+ * THE AGENT'S PAGE (two readers: a person wants feedback and short, scannable responses; an agent wants clear,
+ * unambiguous context and machine-readable formats; both want the next steps): what a
+ * coding agent driving Mason reads first — the commands in the order a project takes them, every answer's shape, the
+ * exit codes and the failure codes — built from the registry and src/cli/command.ts, so it cannot list a command or a
+ * code the CLI does not have.
+ */
+async function agentGuide() {
+    // imported here, not at the top: the registry imports this file for the HELP usage
+    const { STEPS } = await import("./registry.js");
+    const { FAILURE_CODES } = await import("./command.js");
+    const noJson = new Set(["help"]);
+    const w = Math.max(...STEPS.flatMap(([, cs]) => cs.map((c) => c.name.length))) + "mason  --json".length;
+    const commands = STEPS.map(([step, cs]) => `  ${step}\n${cs.map((c) => `    ${`mason ${c.name}${noJson.has(c.name) ? "" : " --json"}`.padEnd(w)}  ${c.summary}`).join("\n")}`).join("\n");
+    const codes = Object.entries(FAILURE_CODES).map(([code, what]) => `    ${code.padEnd(12)} ${what}`).join("\n");
+    return `DRIVING MASON FROM A CODING AGENT — the same commands a person runs, answered as data
+
+  Every command prints for a person, or with --json one JSON object for a program. One prints text alone: mason help
+  (this reference); mason logs --json is the log's lines, or with --requests the request log. Nothing a command prints
+  refers to a file this package does not ship.
+
+  THE COMMANDS, in the order a project takes them (mason --help lists the same):
+${commands}
+
+  EVERY ANSWER is an object with next: [{ command, what }] — the commands to run next, the same a person reads under
+  Next. The other keys are each command's; its help names them (mason help <command>, the --json line). Lists are wrapped
+  ({ tables: […], next }), never bare. mason wait --json blocks until every added table is copied and kept current, then
+  answers once (exit 0), or with error beside the state (exit 1 failed, 124 timed out: nothing stopped, run it again).
+
+  A FAILURE is { "error": { "code", "message", "why", "fix" }, "next": […] } on stdout ("why" and "fix" where there is
+  more to say than the message), and the exit code says it — the same code exits the same wherever it is found:
+    0  done
+    1  failed, with the reason
+    2  the command line is wrong (without --json the usage follows)
+    124  ran out of time: mason wait
+  The codes:
+${codes}
+
+  READING THE OUTPUT: the answer is on stdout, progress on stderr (mason wait's bars; nothing under --json). Colour,
+  wrapping and the spinner are a terminal's alone: a pipe gets plain text, and NO_COLOR turns colour off anywhere.
+  A Postgres tool (psql, a JDBC or ODBC driver, Metabase, Grafana) connects with what mason user add <name> --json
+  answers: url, host, port, database, user, password (printed once; it does not expire) — each added table a table.
+  Where the Postgres API listens is in mason status --json (postgres.url, host, port); never read .mason/config.json
+  for it. A failure that is a bug in mason says so; MASON_DEBUG=1 adds where it happened.
+
+  THE SOURCES FILE, mason/sources/<name>.yml (mason init writes it; one per project for now), is what Mason copies:
+    kind: postgres
+    url: \${DATABASE_URL}          a variable from .env.local, .env or the shell; a literal URL is allowed for local dev
+    tables:
+      orders:                     a bare name is in public; billing.invoices names its schema
+        columns: [id, status]     these columns (the primary key is always copied)
+      customers: {}               every column
+      events:
+        exclude: [raw_body]       every column but these
+      sessions:
+        key: [tenant_id, id]      only for a table with no primary key: the unique columns to copy it by (polling only)
+  mason add and mason remove edit one entry of it, keep the rest as it was, and take effect at once; an edit by hand is
+  read at the next mason start (or the next add or remove). Column types are read from the database at every start.
+
+  A FIRST SESSION, as a shell runs it:
+    mason doctor --json && mason init --json && mason start --json
+    mason tables --json                       choose the tables (rows, key, whether one can be added)
+    mason add orders customers --json && mason wait --json
+    mason user add my-tool --json             the URL and password a tool connects with
+    mason status --json                       each table's copy, and the last day's slow and refused statements
+`;
+}
+const REPO = "https://github.com/embeddable-hq/mason-data/blob/main";
+/**
+ * `mason help production` (the journey from running it locally to deploying it, on the mason-data image): the
+ * README's "On a server", offline and short — what travels, what the server needs, how it runs, how it is upgraded.
+ */
+export function productionGuide() {
+    return `GOING TO PRODUCTION — the project folder (mason/sources/, mason/mason.yml) is all that travels: its sources file
+names the tables the server copies, and its url: names the variable that holds the database's URL. Nothing in .mason/ goes with it; the server makes its own copy and state on its volume.
+
+1. PREPARE THE DATABASE. Mason only reads it. Polling (the default) needs a user that can SELECT the tables, and costs
+   the database a scan of each table per poll; it works on a read replica. Logical replication (MASON_CHANGES=replication)
+   needs wal_level = logical, a user with REPLICATION, and either a publication you name (PG_PUBLICATION, read as it is,
+   never altered) or the right to create one; the slot holds the database's WAL while Mason is down, so cap it with
+   max_slot_wal_keep_size. Try it locally first:
+     mason start --replication                      the change log, as the server reads it
+     mason start --replication --publication app_pub   through a publication the database's owner made
+
+2. RUN IT: one container, ghcr.io/embeddable-hq/mason-data:<version> (the version mason.yml names), running mason serve:
+     DATABASE_URL           the database to copy from (the variable the sources file's url: names)
+     MASON_API_KEY          what the users' passwords are made with (openssl rand -hex 24)
+     MASON_PROJECT          your mason/ folder, mounted (default ./mason)
+     MASON_DATA             the volume whose .mason/ holds the copy and the state (default .)
+     MASON_PG_HOST=0.0.0.0 and TLS: MASON_PG_TLS_CERT / MASON_PG_TLS_KEY, or MASON_PG_SELF_SIGNED_TLS=1, or
+     MASON_PG_PLAIN=1 behind a TLS proxy of your own
+   Docker Compose: ${REPO}/deploy/mason-data/README.md. One container per volume, never two: DuckDB lets one process
+   write its file, so run one replica, with no autoscaling, and stop the old container before the new one starts.
+   GET /healthz on MASON_HEALTH_PORT (4140) answers 503 while it starts and 200 once it serves; the first start takes
+   about 15 seconds before that, then copies the tables.
+
+3. CONNECT THE TOOLS: beside the server (docker compose exec mason …), a user is a password a tool keeps:
+     mason user add metabase                        its URL and password, printed once
+     mason user list
+     mason user remove metabase                     ends it at once, without a restart
+   The tool connects to the Postgres API (5433) as to a read replica, each added table a table. Changing
+   MASON_API_KEY removes every user.
+
+4. CHANGE IT: ship mason/ with your release and restart the container; a table added or removed is copied or dropped
+   at the next start, or beside the server:
+     mason add public.orders                        copied at once, added to the mounted mason/sources/
+     mason remove orders                            taken out of the sources file, its copy dropped
+     mason status                                   each table's copy, the slow and refused statements
+   Upgrade: read the release notes, then move mason.yml and the image together. Removing Mason for good in replication mode:
+   SELECT pg_drop_replication_slot('mason_dev_<name>_<source>') — the slot holds your WAL (mason status names it).
+
+The whole account: ${REPO}/README.md
+`;
+}