@mason-data/cli 0.6.2 → 0.8.0

package/dist/src/cli/doctor.js

@@ -14,6 +14,7 @@ import { theRun } from "./runs.js";
 import { projectSource } from "./project.js";
 import { DEFAULT_PG_PORT } from "../api/postgres/listener.js";
 import { defaultMemoryLimit, machineMemoryBytes, MEMORY_LIMIT_RE } from "../duck/store.js";
+import { probeExtensions } from "../duck/extensions.js";
 const out = (m) => { process.stdout.write(`${m}\n`); };
 /**
  * THE NODE THE PACKAGE NEEDS, major.minor: cli/package.json's `engines`, this check, `mason --help` and the README read
@@ -25,10 +26,12 @@ const out = (m) => { process.stdout.write(`${m}\n`); };
 export const NODE_MIN = "22.13";
 export const DOCTOR_USAGE = `usage: mason doctor [--json]
 
-  Says whether this machine is ready for Mason — Node ${NODE_MIN} or later, memory (Mason's copy uses up to half of it by
-  default; MASON_DUCKDB_MEMORY=4GiB sets it), the ports Mason listens on — with the way out for anything missing. In a
+  Says whether this machine is ready for Mason — Node ${NODE_MIN} or later, memory (the engine uses up to half of it by
+  default; MASON_DUCKDB_MEMORY=4GiB sets it), Mason's postgres and inet extensions (installed with Mason for this
+  platform, so nothing is downloaded; else the first sync downloads them), the ports Mason listens on — with the way
+  out for anything missing. In a
   project's folder it checks the project's database too; with Mason running for it, the run in a few lines instead: its
-  database, whether every added table is copied, the statements of the last day it refused or answered slowly.
+  database, whether every added table is synced, the statements of the last day it refused or answered slowly.
 
     mason doctor
 
@@ -65,10 +68,16 @@ export function memoryCheck(totalBytes = machineMemoryBytes(), env = process.env
     const low = totalBytes < 2 * 1024 ** 3;
     return {
         name: "memory", ok: !low,
-        detail: `${gib} GiB; Mason's copy uses up to ${share.trim()}${set !== undefined ? " (MASON_DUCKDB_MEMORY)" : " (half, the default; MASON_DUCKDB_MEMORY sets it)"}` +
-            (low ? ". Under 2 GiB, a large table's copy and a query starve each other: expect statements refused for memory" : ""),
+        detail: `${gib} GiB; the engine uses up to ${share.trim()}${set !== undefined ? " (MASON_DUCKDB_MEMORY)" : " (half, the default; MASON_DUCKDB_MEMORY sets it)"}` +
+            (low ? ". Under 2 GiB, a large table's first sync and a query starve each other: expect statements refused for memory" : ""),
     };
 }
+/** The copy's extensions (src/duck/extensions.ts): shipped with Mason and loaded by an engine of their own as the run's
+ *  will load them, or not shipped, and why. `probe` is a test's. */
+export async function extensionsCheck(probe = () => probeExtensions()) {
+    const p = await probe();
+    return { name: "extensions", ok: p.ok, detail: p.detail };
+}
 /**
  * Mason's ports in one line, decided by the probe mason start uses (`portHolder`, src/cli/ports.ts: every address, so
  * doctor and start agree on a port another program holds on the wildcard). A taken command port only moves Mason to
@@ -111,11 +120,11 @@ function portOf(url) {
     } // not a URL: no port to compare with
 }
 /** The database a URL names: reachable, its version, what this user can read. */
-export async function databaseCheck(url) {
+export async function databaseCheck(url, o = {}) {
     const parsed = parseSource(url);
     if ("problem" in parsed)
         return { name: "database", ok: false, detail: parsed.problem };
-    const connected = await connectSource(parsed.source);
+    const connected = await connectSource(parsed.source, o);
     if ("failure" in connected)
         return { name: "database", ok: false, detail: `${maskedUrl(parsed.source.url)}: ${connected.failure.message}${connected.failure.hints.map((h) => `\n    - ${h}`).join("")}` };
     let facts;
@@ -153,7 +162,7 @@ export async function runChecks(run) {
     else if (ready.waiting.length)
         checks.push({ name: "project", ok: null, detail: `still at work: ${ready.waiting.join("; ")} (mason wait)` });
     else
-        checks.push({ name: "project", ok: true, detail: picked ? `${picked} table(s) copied and kept current` : "no table added yet (mason tables, then mason add <table>)" });
+        checks.push({ name: "project", ok: true, detail: picked ? `${picked} table(s) synced and kept current` : "no table added yet (mason tables, then mason add <table>)" });
     const slow = await callRun(run, "GET", "/api/slow"), refused = await callRun(run, "GET", "/api/refused");
     const slowN = slow.status === 200 ? slow.body.slow.length : null, refusedN = refused.status === 200 ? refused.body.refused.length : null;
     checks.push(slowN === null || refusedN === null
@@ -168,11 +177,14 @@ export function nextSteps(checks, run, project) {
         return [["mason doctor", `again, once Node ${NODE_MIN} or later is installed`]];
     if (bad("project") && run === null)
         return [["mason doctor", "again, once .mason/config.json is repaired or moved aside (the detail above says how)"]];
+    // the demo database is hosted by the run itself (src/cli/demo.ts): down until mason start makes it
+    if (bad("database") && project?.config.demo === true)
+        return [["mason start", "hosts the demo database, then syncs its tables"]];
     if (bad("database"))
         return [["mason start", `once the connection works (the detail above says how; the URL is in ${project ? `${project.dir}/config.json` : ".mason/config.json"})`]];
     if (project === null)
         return [
-            ["mason init", "in your app's folder: the database from DATABASE_URL in .env.local, .env or the shell"],
+            ["mason init", "in your app's folder: the database from MASON_DATABASE_URL or DATABASE_URL in .env.local, .env or the shell"],
             ["mason init postgres://user:password@localhost:5432/your_app", "or name it"],
         ];
     if (run === null)
@@ -182,7 +194,7 @@ export function nextSteps(checks, run, project) {
     if (bad("project"))
         return [["mason wait", "each problem, with the way out"]];
     if (checks.some((c) => c.name === "project" && /no table added/.test(c.detail)))
-        return [["mason tables", "the database's tables and their columns"], ["mason add <table>", "copy a table into Mason"]];
+        return [["mason tables", "the database's tables and their columns"], ["mason add <table>", "sync a table into Mason"]];
     if (bad("queries"))
         return [["mason status", "the refused and slow statements, each with what to do"]];
     return [["mason user add <name>", "a user for a BI tool or your app: its password and URL, printed once"]];
@@ -219,12 +231,13 @@ export async function doctor(argv, deps = {}) {
     if (run === null) {
         checks.push(nodeCheck());
         checks.push(memoryCheck());
+        checks.push(await extensionsCheck());
         checks.push(await portsCheck([], portHolder, process.env, project?.config.pgPort ?? null));
         if (project) {
             // the database the project's sources file names (src/cli/sources.ts): a file that cannot be read is the failing line
             const src = projectSource(project);
             checks.push("message" in src ? { name: "database", ok: false, detail: `${src.message}${src.why ? `: ${src.why.replace(/\n/g, "; ")}` : ""}. ${src.fix?.split("\n")[0] ?? ""}`.trim() }
-                : await databaseCheck(src.resolved));
+                : await databaseCheck(src.resolved, { demo: project.config.demo === true }));
         }
         // a project here whose config cannot be read: init would refuse, so "Ready: mason init" would send a person nowhere
         if ("code" in which && which.code === "bad_config")
@@ -250,7 +263,7 @@ export async function doctor(argv, deps = {}) {
         out(`  ${mark} ${c.name.padEnd(w)}  ${wrap(c.detail, s.width, 2 + 3 + 1 + w + 2)}`);
     }
     if (run !== null)
-        out(s.dim(`  its copy of the data and its state in ${run.home}. mason status for the details`));
+        out(s.dim(`  its synced data and its state in ${run.home}. mason status for the details`));
     if (run === null || !ready)
         out(nextBlock(next));
     return ready ? 0 : 1;

package/dist/src/cli/guides.js

@@ -7,6 +7,7 @@
  *
  * A guide's name is never a command's (tests/cli-help.test.ts): one word never means two pages.
  */
+import { DEMO_ADDED_TABLES, DEMO_TABLES, DEMO_TABLE_TO_ADD } from "../demo/seed.js";
 /** The guides, by the name `mason help <guide>` takes. */
 export const GUIDES = {
     agent: "driving Mason from a coding agent: the commands in order, --json and its shapes, exit codes, failures",
@@ -29,22 +30,31 @@ async function agentGuide() {
     // imported here, not at the top: the registry imports this file for the HELP usage
     const { STEPS } = await import("./registry.js");
     const { FAILURE_CODES } = await import("./command.js");
-    const noJson = new Set(["help"]);
     const w = Math.max(...STEPS.flatMap(([, cs]) => cs.map((c) => c.name.length))) + "mason  --json".length;
-    const commands = STEPS.map(([step, cs]) => `  ${step}\n${cs.map((c) => `    ${`mason ${c.name}${noJson.has(c.name) ? "" : " --json"}`.padEnd(w)}  ${c.summary}`).join("\n")}`).join("\n");
+    const commands = STEPS.map(([step, cs]) => `  ${step}\n${cs.map((c) => `    ${`mason ${c.name}${Object.hasOwn(c.options, "json") ? " --json" : ""}`.padEnd(w)}  ${c.summary}`).join("\n")}`).join("\n");
     const codes = Object.entries(FAILURE_CODES).map(([code, what]) => `    ${code.padEnd(12)} ${what}`).join("\n");
     return `DRIVING MASON FROM A CODING AGENT — the same commands a person runs, answered as data
 
   Every command prints for a person, or with --json one JSON object for a program. One prints text alone: mason help
-  (this reference); mason logs --json is the log's lines, or with --requests the request log. Nothing a command prints
+  (this reference); mason mcp's stdout is the MCP protocol's, so it takes no --json and says a failure to start on
+  stderr; mason logs --json is the log's entries ({ at, text }), or with --requests the request log, --events the sync's
+  events, --running the statements running now. Nothing a command prints
   refers to a file this package does not ship.
 
+  AN AGENT WITH MCP (Claude Code, Claude Desktop, any MCP client) adds Mason as an MCP server first, then asks with
+  its tools rather than through a shell: tables (each synced table's rows and how current it is), describe, query (rows with
+  their Postgres types, or the refusal with what to write instead) and explain. In the project's folder, Mason running:
+    mason user add claude --json              its url, printed once
+    claude mcp add mason --env MASON_URL="<url>" -- mason mcp
+  (the server's name before --env, which takes every value after it). mason help mcp says the rest: Claude Desktop's
+  config, a mason serve elsewhere.
+
   THE COMMANDS, in the order a project takes them (mason --help lists the same):
 ${commands}
 
   EVERY ANSWER is an object with next: [{ command, what }] — the commands to run next, the same a person reads under
   Next. The other keys are each command's; its help names them (mason help <command>, the --json line). Lists are wrapped
-  ({ tables: […], next }), never bare. mason wait --json blocks until every added table is copied and kept current, then
+  ({ tables: […], next }), never bare. mason wait --json blocks until every added table is synced and kept current, then
   answers once (exit 0), or with error beside the state (exit 1 failed, 124 timed out: nothing stopped, run it again).
 
   A FAILURE is { "error": { "code", "message", "why", "fix" }, "next": […] } on stdout ("why" and "fix" where there is
@@ -63,69 +73,131 @@ ${codes}
   Where the Postgres API listens is in mason status --json (postgres.url, host, port); never read .mason/config.json
   for it. A failure that is a bug in mason says so; MASON_DEBUG=1 adds where it happened.
 
-  THE SOURCES FILE, mason/sources/<name>.yml (mason init writes it; one per project for now), is what Mason copies:
+  THE SOURCES FILE, mason/sources/<name>.yml (mason init writes it; one per project for now), is what Mason syncs:
     kind: postgres
     url: \${DATABASE_URL}          a variable from .env.local, .env or the shell; a literal URL is allowed for local dev
+    database: production          optional: the database name a tool connects with (none: mason), to keep the name it has
+    sync: off                     how each table is kept current after its first sync: off (the default; mason sync
+                                  <table> reads it again), poll <interval> (poll 5s, poll 5m, poll 1h; 1s to 24h) or
+                                  replication (the change log: a primary key, wal_level = logical)
+    publication: \${PG_PUBLICATION}  optional: the publication replication reads, its owner's, never altered
+                                     (publication: null: Mason's own, past one an earlier start recorded)
     tables:
       orders:                     a bare name is in public; billing.invoices names its schema
-        columns: [id, status]     these columns (the primary key is always copied)
+        columns: [id, status]     these columns (the primary key is always synced)
+        sync: poll 30s            this table's own: polled every 30 seconds (mason add orders --sync "poll 30s")
       customers: {}               every column
       events:
         exclude: [raw_body]       every column but these
       sessions:
-        key: [tenant_id, id]      only for a table with no primary key: the unique columns to copy it by (polling only)
-  mason add and mason remove edit one entry of it, keep the rest as it was, and take effect at once; an edit by hand is
-  read at the next mason start (or the next add or remove). Column types are read from the database at every start.
+        key: [tenant_id, id]      only for a table with no primary key: the unique columns to key it by (polling only)
+  mason add and mason remove edit one entry of it and keep the rest as it was, Mason running or not: with it running
+  they take effect at once, else at mason start. An edit by hand is read at the next mason start (or the next add or
+  remove). Column types are read from the database at every start. A change of a table's sync: keeps its synced data and
+  catches it up by one poll. mason status --json says each table's kept: { sync, syncedAt, nextAt } — how old its
+  synced data is; a table synced days ago answers today's count(*) as of then.
 
   A FIRST SESSION, as a shell runs it:
-    mason doctor --json && mason init --json && mason start --json
+    mason doctor --json && mason init --json
     mason tables --json                       choose the tables (rows, key, whether one can be added)
-    mason add orders customers --json && mason wait --json
+    mason add orders customers --json         added to the sources file, checked against the database
+    mason start --json && mason wait --json   Mason in the background; until every added table is synced
+    mason sync orders --json                  read what changed in it since (a table whose sync is off: only so)
     mason user add my-tool --json             the URL and password a tool connects with
-    mason status --json                       each table's copy, and the last day's slow and refused statements
+    mason status --json                       each synced table, and the last day's slow, refused and unsupported statements
+  WATCHING IT, once it runs (every one with --json; --since and --until take an age, 30m or 2h or 7d, or an ISO 8601 time WITH
+  its zone):
+    mason status --json                       recentErrors: the sync's latest failures and the statements latest refused, failed or cancelled
+    mason logs --requests --failed --json     every statement a tool sent that was not answered, with its SQLSTATE and message
+    mason logs --events --failed --json       the sync's failures: a first sync, a poll, the stream, the database out of reach, a restart
+    mason logs --running --json               the statements running now, one per session
+    mason cancel <session> --json             stops that session's statement, as a client's cancel does (its client gets a "canceling statement" error)
+    mason usage --since 7d --step 1h --json  the statements, the synced data's size and age, the memory over time, a sample a minute
+  With no database to hand, in a new folder: mason init demo --json && mason start --json — a demo database Mason
+  hosts itself (${DEMO_TABLES.join(", ")}), changed every few seconds, polled every 5 s (its sync: poll 5s). Init adds
+  ${DEMO_ADDED_TABLES.join(", ")}; ${DEMO_TABLE_TO_ADD} is left to add: mason add ${DEMO_TABLE_TO_ADD} --json && mason wait --json, after mason start.
 `;
 }
 const REPO = "https://github.com/embeddable-hq/mason-data/blob/main";
 /**
  * `mason help production` (the journey from running it locally to deploying it, on the mason-data image): the
- * README's "On a server", offline and short — what travels, what the server needs, how it runs, how it is upgraded.
+ * README's "Going to production", offline and short — what travels, what the server needs, how it runs, how it is upgraded.
  */
 export function productionGuide() {
     return `GOING TO PRODUCTION — the project folder (mason/sources/, mason/mason.yml) is all that travels: its sources file
-names the tables the server copies, and its url: names the variable that holds the database's URL. Nothing in .mason/ goes with it; the server makes its own copy and state on its volume.
+names the tables the server syncs, and its url: names the variable that holds the database's URL. Nothing in .mason/ goes with it; the server makes its own synced data and state on its volume.
 
-1. PREPARE THE DATABASE. Mason only reads it. Polling (the default) needs a user that can SELECT the tables, and costs
-   the database a scan of each table per poll; it works on a read replica. Logical replication (MASON_CHANGES=replication)
-   needs wal_level = logical, a user with REPLICATION, and either a publication you name (PG_PUBLICATION, read as it is,
-   never altered) or the right to create one; the slot holds the database's WAL while Mason is down, so cap it with
-   max_slot_wal_keep_size. Try it locally first:
-     mason start --replication                      the change log, as the server reads it
-     mason start --replication --publication app_pub   through a publication the database's owner made
+1. PREPARE THE DATABASE. Mason only reads it, as a user that can SELECT the tables. How each table is kept current
+   after its first sync is the sources file's sync:, committed with it, so the server runs what you ran — at the top
+   for every table, and on an entry for that one:
+     sync: off                                      the file's default for every table
+     tables:
+       orders: { sync: replication }                this one from the change log, within seconds
+       customers: { sync: poll 5m }                 this one polled every 5 minutes
+       events: {}                                   off, as the top says
+   off (the default mason init writes): after the first sync Mason reads the table again only when you run mason sync
+   <table> (before a report, from cron, at the end of a deploy) — nothing Mason does to your database is a hidden
+   default. poll <interval> (1s to 24h): a SELECT that scans the table whole, at that interval, stretched to four
+   times the last poll's own time, so the polls never take more than a fifth of the database's time; it works on a
+   read replica. replication: the change log, through one slot and one publication for the project; it needs
+   wal_level = logical, a user with REPLICATION (or rds_replication), a primary key (a key: table is polled), and
+   either a publication you name (publication: \${PG_PUBLICATION} at the top of the sources file, read as it is, never
+   altered: it must carry every replicated table; a read replica needs one) or the right to create Mason's own, which
+   carries the replicated tables alone. The slot holds the database's WAL while Mason is down, so cap it on the
+   database with max_slot_wal_keep_size (Postgres 13+); it is dropped, verified gone, when no table's sync is
+   replication any more, and at mason destroy. The flags --replication, --poll and --publication of mason start, and
+   MASON_CHANGES, are gone: each answers with the line to write in the sources file. A project that streams, under a
+   file with no sync: anywhere, is refused rather than dropped (sync: replication keeps the slot; sync: off or
+   sync: poll <interval> confirms the drop), and so is one with a recorded publication that nothing names now
+   (publication: <name>, or publication: null for Mason's own, with PG_PUBLICATION unset). Try it locally first:
+     mason add orders --sync replication             that table from the change log, as the server reads it
+     mason sync customers                           any table read again now, whatever its sync:
 
 2. RUN IT: one container, ghcr.io/embeddable-hq/mason-data:<version> (the version mason.yml names), running mason serve:
-     DATABASE_URL           the database to copy from (the variable the sources file's url: names)
+     DATABASE_URL           the database to sync from (the variable the sources file's url: names)
      MASON_API_KEY          what the users' passwords are made with (openssl rand -hex 24)
      MASON_PROJECT          your mason/ folder, mounted (default ./mason)
-     MASON_DATA             the volume whose .mason/ holds the copy and the state (default .)
+     MASON_DATA             the volume whose .mason/ holds the synced data and the state (default .)
      MASON_PG_HOST=0.0.0.0 and TLS: MASON_PG_TLS_CERT / MASON_PG_TLS_KEY, or MASON_PG_SELF_SIGNED_TLS=1, or
      MASON_PG_PLAIN=1 behind a TLS proxy of your own
-   Docker Compose: ${REPO}/deploy/mason-data/README.md. One container per volume, never two: DuckDB lets one process
+   Docker Compose: ${REPO}/deploy/mason-data/README.md. One container per volume, never two: the synced data lets one process
    write its file, so run one replica, with no autoscaling, and stop the old container before the new one starts.
    GET /healthz on MASON_HEALTH_PORT (4140) answers 503 while it starts and 200 once it serves; the first start takes
-   about 15 seconds before that, then copies the tables.
+   about 15 seconds before that, then syncs the tables. GET /metrics on the same port is for Prometheus: numbers only
+   (tables by phase, lag, ages, sizes, statements), never a table's or a user's name, so it needs no secret.
+   MEMORY: the engine may use half the container's memory limit by default (else half the machine's RAM); the other
+   half is the page cache, Node and the catalog (~820 MB). Give the container enough that the page cache holds the
+   columns your dashboards read: a query over a column it does not hold runs at the disk's speed. The benchmark's synced data
+   was about a fifth of its tables in Postgres (10.1 GiB to 1.8 GiB, on a 15 GiB machine); mason status says yours.
+   To set it, MASON_DUCKDB_MEMORY=4GB with the container's limit about twice that. A statement past it fails alone
+   (53200) and the run goes on; a value near the container's limit is an OOM kill.
+   CPU: memory first, then cores. The benchmark's machine (4 vCPU Graviton3, 15 GiB) answered 32 clients at 10 M rows,
+   12 statements a second at a p95 of 3.5 s, as many a second as at 8 clients: more clients waited longer. A first sync's
+   readers take about a core each while it runs (MASON_COPY_PARALLEL, default 3). The image is arm64 (the measured
+   one) and amd64. Which settings to decide: ${REPO}/deploy/mason-data/README.md#settings
 
 3. CONNECT THE TOOLS: beside the server (docker compose exec mason …), a user is a password a tool keeps:
      mason user add metabase                        its URL and password, printed once
+     mason user add agent --role agent              a user held to a role of mason/roles.yml: the tables and columns it reads
+     mason user add acme --role tenant --set tenant=42   and the rows: a role whose where: reads :tenant
      mason user list
      mason user remove metabase                     ends it at once, without a restart
-   The tool connects to the Postgres API (5433) as to a read replica, each added table a table. Changing
-   MASON_API_KEY removes every user.
+   The tool connects to the Postgres API (5433) as to a read replica, each added table a table, in the database
+   mason: a tool moving from your Postgres changes its host, port and database name, or database: <name> in the sources
+   file keeps its name (MASON_PG_DATABASE on a server whose sources file has no such line). Changing
+   MASON_API_KEY removes every user. mason/roles.yml ships with mason/ (mason help user shows the file): a user with
+   a role is refused anything past it as Postgres refuses it (42501), and information_schema and mason mcp list it only
+   what it may read; a user added without --role reads as the file's default: (mason init writes default:
+   everything, tables: all — every synced table), and nothing when the file names no default. A role's where: and
+   through: filter its tables' rows by the values each user carries (--set), every read of them filtered; a user
+   missing a value is refused at every statement, never answered with no rows. A roles file from before this release
+   names no default: write default: <role> at its top before upgrading, or its roleless users read nothing.
 
-4. CHANGE IT: ship mason/ with your release and restart the container; a table added or removed is copied or dropped
+4. CHANGE IT: ship mason/ with your release and restart the container; a table added or removed is synced or dropped
    at the next start, or beside the server:
-     mason add public.orders                        copied at once, added to the mounted mason/sources/
-     mason remove orders                            taken out of the sources file, its copy dropped
-     mason status                                   each table's copy, the slow and refused statements
+     mason add public.orders                        synced at once, added to the mounted mason/sources/
+     mason remove orders                            taken out of the sources file, its synced data dropped
+     mason status                                   each synced table, the statements, the recent errors
    Upgrade: read the release notes, then move mason.yml and the image together. Removing Mason for good in replication mode:
    SELECT pg_drop_replication_slot('mason_dev_<name>_<source>') — the slot holds your WAL (mason status names it).