@mason-data/cli 0.7.0 → 0.8.0

package/dist/src/cli/doctor.js

@@ -14,6 +14,7 @@ import { theRun } from "./runs.js";
 import { projectSource } from "./project.js";
 import { DEFAULT_PG_PORT } from "../api/postgres/listener.js";
 import { defaultMemoryLimit, machineMemoryBytes, MEMORY_LIMIT_RE } from "../duck/store.js";
+import { probeExtensions } from "../duck/extensions.js";
 const out = (m) => { process.stdout.write(`${m}\n`); };
 /**
  * THE NODE THE PACKAGE NEEDS, major.minor: cli/package.json's `engines`, this check, `mason --help` and the README read
@@ -25,10 +26,12 @@ const out = (m) => { process.stdout.write(`${m}\n`); };
 export const NODE_MIN = "22.13";
 export const DOCTOR_USAGE = `usage: mason doctor [--json]
 
-  Says whether this machine is ready for Mason — Node ${NODE_MIN} or later, memory (Mason's copy uses up to half of it by
-  default; MASON_DUCKDB_MEMORY=4GiB sets it), the ports Mason listens on — with the way out for anything missing. In a
+  Says whether this machine is ready for Mason — Node ${NODE_MIN} or later, memory (the engine uses up to half of it by
+  default; MASON_DUCKDB_MEMORY=4GiB sets it), Mason's postgres and inet extensions (installed with Mason for this
+  platform, so nothing is downloaded; else the first sync downloads them), the ports Mason listens on — with the way
+  out for anything missing. In a
   project's folder it checks the project's database too; with Mason running for it, the run in a few lines instead: its
-  database, whether every added table is copied, the statements of the last day it refused or answered slowly.
+  database, whether every added table is synced, the statements of the last day it refused or answered slowly.
 
     mason doctor
 
@@ -65,10 +68,16 @@ export function memoryCheck(totalBytes = machineMemoryBytes(), env = process.env
     const low = totalBytes < 2 * 1024 ** 3;
     return {
         name: "memory", ok: !low,
-        detail: `${gib} GiB; Mason's copy uses up to ${share.trim()}${set !== undefined ? " (MASON_DUCKDB_MEMORY)" : " (half, the default; MASON_DUCKDB_MEMORY sets it)"}` +
-            (low ? ". Under 2 GiB, a large table's copy and a query starve each other: expect statements refused for memory" : ""),
+        detail: `${gib} GiB; the engine uses up to ${share.trim()}${set !== undefined ? " (MASON_DUCKDB_MEMORY)" : " (half, the default; MASON_DUCKDB_MEMORY sets it)"}` +
+            (low ? ". Under 2 GiB, a large table's first sync and a query starve each other: expect statements refused for memory" : ""),
     };
 }
+/** The copy's extensions (src/duck/extensions.ts): shipped with Mason and loaded by an engine of their own as the run's
+ *  will load them, or not shipped, and why. `probe` is a test's. */
+export async function extensionsCheck(probe = () => probeExtensions()) {
+    const p = await probe();
+    return { name: "extensions", ok: p.ok, detail: p.detail };
+}
 /**
  * Mason's ports in one line, decided by the probe mason start uses (`portHolder`, src/cli/ports.ts: every address, so
  * doctor and start agree on a port another program holds on the wildcard). A taken command port only moves Mason to
@@ -153,7 +162,7 @@ export async function runChecks(run) {
     else if (ready.waiting.length)
         checks.push({ name: "project", ok: null, detail: `still at work: ${ready.waiting.join("; ")} (mason wait)` });
     else
-        checks.push({ name: "project", ok: true, detail: picked ? `${picked} table(s) copied and kept current` : "no table added yet (mason tables, then mason add <table>)" });
+        checks.push({ name: "project", ok: true, detail: picked ? `${picked} table(s) synced and kept current` : "no table added yet (mason tables, then mason add <table>)" });
     const slow = await callRun(run, "GET", "/api/slow"), refused = await callRun(run, "GET", "/api/refused");
     const slowN = slow.status === 200 ? slow.body.slow.length : null, refusedN = refused.status === 200 ? refused.body.refused.length : null;
     checks.push(slowN === null || refusedN === null
@@ -170,12 +179,12 @@ export function nextSteps(checks, run, project) {
         return [["mason doctor", "again, once .mason/config.json is repaired or moved aside (the detail above says how)"]];
     // the demo database is hosted by the run itself (src/cli/demo.ts): down until mason start makes it
     if (bad("database") && project?.config.demo === true)
-        return [["mason start", "hosts the demo database, then copies its tables"]];
+        return [["mason start", "hosts the demo database, then syncs its tables"]];
     if (bad("database"))
         return [["mason start", `once the connection works (the detail above says how; the URL is in ${project ? `${project.dir}/config.json` : ".mason/config.json"})`]];
     if (project === null)
         return [
-            ["mason init", "in your app's folder: the database from DATABASE_URL in .env.local, .env or the shell"],
+            ["mason init", "in your app's folder: the database from MASON_DATABASE_URL or DATABASE_URL in .env.local, .env or the shell"],
             ["mason init postgres://user:password@localhost:5432/your_app", "or name it"],
         ];
     if (run === null)
@@ -185,7 +194,7 @@ export function nextSteps(checks, run, project) {
     if (bad("project"))
         return [["mason wait", "each problem, with the way out"]];
     if (checks.some((c) => c.name === "project" && /no table added/.test(c.detail)))
-        return [["mason tables", "the database's tables and their columns"], ["mason add <table>", "copy a table into Mason"]];
+        return [["mason tables", "the database's tables and their columns"], ["mason add <table>", "sync a table into Mason"]];
     if (bad("queries"))
         return [["mason status", "the refused and slow statements, each with what to do"]];
     return [["mason user add <name>", "a user for a BI tool or your app: its password and URL, printed once"]];
@@ -222,6 +231,7 @@ export async function doctor(argv, deps = {}) {
     if (run === null) {
         checks.push(nodeCheck());
         checks.push(memoryCheck());
+        checks.push(await extensionsCheck());
         checks.push(await portsCheck([], portHolder, process.env, project?.config.pgPort ?? null));
         if (project) {
             // the database the project's sources file names (src/cli/sources.ts): a file that cannot be read is the failing line
@@ -253,7 +263,7 @@ export async function doctor(argv, deps = {}) {
         out(`  ${mark} ${c.name.padEnd(w)}  ${wrap(c.detail, s.width, 2 + 3 + 1 + w + 2)}`);
     }
     if (run !== null)
-        out(s.dim(`  its copy of the data and its state in ${run.home}. mason status for the details`));
+        out(s.dim(`  its synced data and its state in ${run.home}. mason status for the details`));
     if (run === null || !ready)
         out(nextBlock(next));
     return ready ? 0 : 1;

package/dist/src/cli/guides.js

@@ -37,11 +37,12 @@ async function agentGuide() {
 
   Every command prints for a person, or with --json one JSON object for a program. One prints text alone: mason help
   (this reference); mason mcp's stdout is the MCP protocol's, so it takes no --json and says a failure to start on
-  stderr; mason logs --json is the log's lines, or with --requests the request log. Nothing a command prints
+  stderr; mason logs --json is the log's entries ({ at, text }), or with --requests the request log, --events the sync's
+  events, --running the statements running now. Nothing a command prints
   refers to a file this package does not ship.
 
   AN AGENT WITH MCP (Claude Code, Claude Desktop, any MCP client) adds Mason as an MCP server first, then asks with
-  its tools rather than through a shell: tables (each copy's rows and how current it is), describe, query (rows with
+  its tools rather than through a shell: tables (each synced table's rows and how current it is), describe, query (rows with
   their Postgres types, or the refusal with what to write instead) and explain. In the project's folder, Mason running:
     mason user add claude --json              its url, printed once
     claude mcp add mason --env MASON_URL="<url>" -- mason mcp
@@ -53,7 +54,7 @@ ${commands}
 
   EVERY ANSWER is an object with next: [{ command, what }] — the commands to run next, the same a person reads under
   Next. The other keys are each command's; its help names them (mason help <command>, the --json line). Lists are wrapped
-  ({ tables: […], next }), never bare. mason wait --json blocks until every added table is copied and kept current, then
+  ({ tables: […], next }), never bare. mason wait --json blocks until every added table is synced and kept current, then
   answers once (exit 0), or with error beside the state (exit 1 failed, 124 timed out: nothing stopped, run it again).
 
   A FAILURE is { "error": { "code", "message", "why", "fix" }, "next": […] } on stdout ("why" and "fix" where there is
@@ -72,30 +73,48 @@ ${codes}
   Where the Postgres API listens is in mason status --json (postgres.url, host, port); never read .mason/config.json
   for it. A failure that is a bug in mason says so; MASON_DEBUG=1 adds where it happened.
 
-  THE SOURCES FILE, mason/sources/<name>.yml (mason init writes it; one per project for now), is what Mason copies:
+  THE SOURCES FILE, mason/sources/<name>.yml (mason init writes it; one per project for now), is what Mason syncs:
     kind: postgres
     url: \${DATABASE_URL}          a variable from .env.local, .env or the shell; a literal URL is allowed for local dev
+    database: production          optional: the database name a tool connects with (none: mason), to keep the name it has
+    sync: off                     how each table is kept current after its first sync: off (the default; mason sync
+                                  <table> reads it again), poll <interval> (poll 5s, poll 5m, poll 1h; 1s to 24h) or
+                                  replication (the change log: a primary key, wal_level = logical)
+    publication: \${PG_PUBLICATION}  optional: the publication replication reads, its owner's, never altered
+                                     (publication: null: Mason's own, past one an earlier start recorded)
     tables:
       orders:                     a bare name is in public; billing.invoices names its schema
-        columns: [id, status]     these columns (the primary key is always copied)
+        columns: [id, status]     these columns (the primary key is always synced)
+        sync: poll 30s            this table's own: polled every 30 seconds (mason add orders --sync "poll 30s")
       customers: {}               every column
       events:
         exclude: [raw_body]       every column but these
       sessions:
-        key: [tenant_id, id]      only for a table with no primary key: the unique columns to copy it by (polling only)
+        key: [tenant_id, id]      only for a table with no primary key: the unique columns to key it by (polling only)
   mason add and mason remove edit one entry of it and keep the rest as it was, Mason running or not: with it running
   they take effect at once, else at mason start. An edit by hand is read at the next mason start (or the next add or
-  remove). Column types are read from the database at every start.
+  remove). Column types are read from the database at every start. A change of a table's sync: keeps its synced data and
+  catches it up by one poll. mason status --json says each table's kept: { sync, syncedAt, nextAt } — how old its
+  synced data is; a table synced days ago answers today's count(*) as of then.
 
   A FIRST SESSION, as a shell runs it:
     mason doctor --json && mason init --json
     mason tables --json                       choose the tables (rows, key, whether one can be added)
     mason add orders customers --json         added to the sources file, checked against the database
-    mason start --json && mason wait --json   Mason in the background; until every added table is copied
+    mason start --json && mason wait --json   Mason in the background; until every added table is synced
+    mason sync orders --json                  read what changed in it since (a table whose sync is off: only so)
     mason user add my-tool --json             the URL and password a tool connects with
-    mason status --json                       each table's copy, and the last day's slow and refused statements
+    mason status --json                       each synced table, and the last day's slow, refused and unsupported statements
+  WATCHING IT, once it runs (every one with --json; --since and --until take an age, 30m or 2h or 7d, or an ISO 8601 time WITH
+  its zone):
+    mason status --json                       recentErrors: the sync's latest failures and the statements latest refused, failed or cancelled
+    mason logs --requests --failed --json     every statement a tool sent that was not answered, with its SQLSTATE and message
+    mason logs --events --failed --json       the sync's failures: a first sync, a poll, the stream, the database out of reach, a restart
+    mason logs --running --json               the statements running now, one per session
+    mason cancel <session> --json             stops that session's statement, as a client's cancel does (its client gets a "canceling statement" error)
+    mason usage --since 7d --step 1h --json  the statements, the synced data's size and age, the memory over time, a sample a minute
   With no database to hand, in a new folder: mason init demo --json && mason start --json — a demo database Mason
-  hosts itself (${DEMO_TABLES.join(", ")}), changed every few seconds, read by polling only. Init adds
+  hosts itself (${DEMO_TABLES.join(", ")}), changed every few seconds, polled every 5 s (its sync: poll 5s). Init adds
   ${DEMO_ADDED_TABLES.join(", ")}; ${DEMO_TABLE_TO_ADD} is left to add: mason add ${DEMO_TABLE_TO_ADD} --json && mason wait --json, after mason start.
 `;
 }
@@ -106,50 +125,79 @@ const REPO = "https://github.com/embeddable-hq/mason-data/blob/main";
  */
 export function productionGuide() {
     return `GOING TO PRODUCTION — the project folder (mason/sources/, mason/mason.yml) is all that travels: its sources file
-names the tables the server copies, and its url: names the variable that holds the database's URL. Nothing in .mason/ goes with it; the server makes its own copy and state on its volume.
+names the tables the server syncs, and its url: names the variable that holds the database's URL. Nothing in .mason/ goes with it; the server makes its own synced data and state on its volume.
 
-1. PREPARE THE DATABASE. Mason only reads it. Polling (the default) needs a user that can SELECT the tables, and costs
-   the database a scan of each table per poll; it works on a read replica. Logical replication (MASON_CHANGES=replication)
-   needs wal_level = logical, a user with REPLICATION, and either a publication you name (PG_PUBLICATION, read as it is,
-   never altered) or the right to create one; the slot holds the database's WAL while Mason is down, so cap it with
-   max_slot_wal_keep_size. Try it locally first:
-     mason start --replication                      the change log, as the server reads it
-     mason start --replication --publication app_pub   through a publication the database's owner made
+1. PREPARE THE DATABASE. Mason only reads it, as a user that can SELECT the tables. How each table is kept current
+   after its first sync is the sources file's sync:, committed with it, so the server runs what you ran — at the top
+   for every table, and on an entry for that one:
+     sync: off                                      the file's default for every table
+     tables:
+       orders: { sync: replication }                this one from the change log, within seconds
+       customers: { sync: poll 5m }                 this one polled every 5 minutes
+       events: {}                                   off, as the top says
+   off (the default mason init writes): after the first sync Mason reads the table again only when you run mason sync
+   <table> (before a report, from cron, at the end of a deploy) — nothing Mason does to your database is a hidden
+   default. poll <interval> (1s to 24h): a SELECT that scans the table whole, at that interval, stretched to four
+   times the last poll's own time, so the polls never take more than a fifth of the database's time; it works on a
+   read replica. replication: the change log, through one slot and one publication for the project; it needs
+   wal_level = logical, a user with REPLICATION (or rds_replication), a primary key (a key: table is polled), and
+   either a publication you name (publication: \${PG_PUBLICATION} at the top of the sources file, read as it is, never
+   altered: it must carry every replicated table; a read replica needs one) or the right to create Mason's own, which
+   carries the replicated tables alone. The slot holds the database's WAL while Mason is down, so cap it on the
+   database with max_slot_wal_keep_size (Postgres 13+); it is dropped, verified gone, when no table's sync is
+   replication any more, and at mason destroy. The flags --replication, --poll and --publication of mason start, and
+   MASON_CHANGES, are gone: each answers with the line to write in the sources file. A project that streams, under a
+   file with no sync: anywhere, is refused rather than dropped (sync: replication keeps the slot; sync: off or
+   sync: poll <interval> confirms the drop), and so is one with a recorded publication that nothing names now
+   (publication: <name>, or publication: null for Mason's own, with PG_PUBLICATION unset). Try it locally first:
+     mason add orders --sync replication             that table from the change log, as the server reads it
+     mason sync customers                           any table read again now, whatever its sync:
 
 2. RUN IT: one container, ghcr.io/embeddable-hq/mason-data:<version> (the version mason.yml names), running mason serve:
-     DATABASE_URL           the database to copy from (the variable the sources file's url: names)
+     DATABASE_URL           the database to sync from (the variable the sources file's url: names)
      MASON_API_KEY          what the users' passwords are made with (openssl rand -hex 24)
      MASON_PROJECT          your mason/ folder, mounted (default ./mason)
-     MASON_DATA             the volume whose .mason/ holds the copy and the state (default .)
+     MASON_DATA             the volume whose .mason/ holds the synced data and the state (default .)
      MASON_PG_HOST=0.0.0.0 and TLS: MASON_PG_TLS_CERT / MASON_PG_TLS_KEY, or MASON_PG_SELF_SIGNED_TLS=1, or
      MASON_PG_PLAIN=1 behind a TLS proxy of your own
-   Docker Compose: ${REPO}/deploy/mason-data/README.md. One container per volume, never two: DuckDB lets one process
+   Docker Compose: ${REPO}/deploy/mason-data/README.md. One container per volume, never two: the synced data lets one process
    write its file, so run one replica, with no autoscaling, and stop the old container before the new one starts.
    GET /healthz on MASON_HEALTH_PORT (4140) answers 503 while it starts and 200 once it serves; the first start takes
-   about 15 seconds before that, then copies the tables.
-   MEMORY: the copy may use half the container's memory limit by default (else half the machine's RAM); the other
+   about 15 seconds before that, then syncs the tables. GET /metrics on the same port is for Prometheus: numbers only
+   (tables by phase, lag, ages, sizes, statements), never a table's or a user's name, so it needs no secret.
+   MEMORY: the engine may use half the container's memory limit by default (else half the machine's RAM); the other
    half is the page cache, Node and the catalog (~820 MB). Give the container enough that the page cache holds the
-   columns your dashboards read: a query over a column it does not hold runs at the disk's speed. The benchmark's copy
+   columns your dashboards read: a query over a column it does not hold runs at the disk's speed. The benchmark's synced data
    was about a fifth of its tables in Postgres (10.1 GiB to 1.8 GiB, on a 15 GiB machine); mason status says yours.
    To set it, MASON_DUCKDB_MEMORY=4GB with the container's limit about twice that. A statement past it fails alone
    (53200) and the run goes on; a value near the container's limit is an OOM kill.
    CPU: memory first, then cores. The benchmark's machine (4 vCPU Graviton3, 15 GiB) answered 32 clients at 10 M rows,
-   12 statements a second at a p95 of 3.5 s, as many a second as at 8 clients: more clients waited longer. A copy's
+   12 statements a second at a p95 of 3.5 s, as many a second as at 8 clients: more clients waited longer. A first sync's
    readers take about a core each while it runs (MASON_COPY_PARALLEL, default 3). The image is arm64 (the measured
    one) and amd64. Which settings to decide: ${REPO}/deploy/mason-data/README.md#settings
 
 3. CONNECT THE TOOLS: beside the server (docker compose exec mason …), a user is a password a tool keeps:
      mason user add metabase                        its URL and password, printed once
+     mason user add agent --role agent              a user held to a role of mason/roles.yml: the tables and columns it reads
+     mason user add acme --role tenant --set tenant=42   and the rows: a role whose where: reads :tenant
      mason user list
      mason user remove metabase                     ends it at once, without a restart
-   The tool connects to the Postgres API (5433) as to a read replica, each added table a table. Changing
-   MASON_API_KEY removes every user.
+   The tool connects to the Postgres API (5433) as to a read replica, each added table a table, in the database
+   mason: a tool moving from your Postgres changes its host, port and database name, or database: <name> in the sources
+   file keeps its name (MASON_PG_DATABASE on a server whose sources file has no such line). Changing
+   MASON_API_KEY removes every user. mason/roles.yml ships with mason/ (mason help user shows the file): a user with
+   a role is refused anything past it as Postgres refuses it (42501), and information_schema and mason mcp list it only
+   what it may read; a user added without --role reads as the file's default: (mason init writes default:
+   everything, tables: all — every synced table), and nothing when the file names no default. A role's where: and
+   through: filter its tables' rows by the values each user carries (--set), every read of them filtered; a user
+   missing a value is refused at every statement, never answered with no rows. A roles file from before this release
+   names no default: write default: <role> at its top before upgrading, or its roleless users read nothing.
 
-4. CHANGE IT: ship mason/ with your release and restart the container; a table added or removed is copied or dropped
+4. CHANGE IT: ship mason/ with your release and restart the container; a table added or removed is synced or dropped
    at the next start, or beside the server:
-     mason add public.orders                        copied at once, added to the mounted mason/sources/
-     mason remove orders                            taken out of the sources file, its copy dropped
-     mason status                                   each table's copy, the slow and refused statements
+     mason add public.orders                        synced at once, added to the mounted mason/sources/
+     mason remove orders                            taken out of the sources file, its synced data dropped
+     mason status                                   each synced table, the statements, the recent errors
    Upgrade: read the release notes, then move mason.yml and the image together. Removing Mason for good in replication mode:
    SELECT pg_drop_replication_slot('mason_dev_<name>_<source>') — the slot holds your WAL (mason status names it).