@mason-data/cli 0.8.0 → 0.10.0

package/dist/src/cli/generated-files.js

@@ -0,0 +1,143 @@
+/**
+ * THE PROJECT'S GENERATED TABLES (issue 124): a folder beside the sources file, one file per table, the project's to
+ * commit and review. The file's name is the table's: `requests.yml` is `public.requests`, `reports.daily.yml` is
+ * `reports.daily`.
+ *
+ *   kind: scheduled               what the table promises: `scheduled` (any SELECT, up to target_lag behind its base)
+ *   target_lag: 5m                how far behind it may be
+ *   sql: |                        what it holds
+ *     SELECT tenant_id, count(*) AS requests FROM events GROUP BY 1
+ *   order_by: tenant_id           optional: what follows ORDER BY, over its columns, one SQL string
+ *   comment: Requests per tenant. optional: COMMENT ON TABLE
+ *
+ * Read here for its shape alone: each mistake at its key, with the reason and the way out, all of a file's at once. What
+ * its SQL means is the catalog's to say (src/duck/generated-plan.ts). A mistake never stops the run or another table,
+ * unlike a sources file's: a table already deployed keeps serving what it was built from (src/duck/generated.ts).
+ *
+ * `GENERATED_KEYS` is the one list of keys: a key the reader takes and the guide says (`mason help generated`), so neither
+ * can name one the other does not.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { parseDocument } from "yaml";
+import { didYouMean } from "../lib/did-you-mean.js";
+import { parseLag } from "../lib/table-sync.js";
+import { shownPath, tableNameProblem } from "./sources.js";
+/** Where a project's generated tables are: `<folder>/generated_tables/`, beside `sources/`. */
+export const GENERATED_DIR = "generated_tables";
+export const generatedDir = (folder) => path.join(folder, GENERATED_DIR);
+/** Every key a file takes, and what it says, as a person reads it. */
+export const GENERATED_KEYS = {
+    kind: "required: how the table stays current. scheduled: any SELECT, up to target_lag behind its base. live is not available yet",
+    sql: "required: the one SELECT that gives the table, over the tables the sources file syncs",
+    target_lag: "required for scheduled: how far behind its base it may be, like 30s, 5m or 1h (at least 1s, at most 24h). Mason looks at each table every 5s, so a table under 5s can be up to 5s behind",
+    order_by: "optional: the table's rows are written in this order, as what follows ORDER BY, over its columns, one SQL string",
+    comment: "optional: the table's comment, as COMMENT ON TABLE",
+};
+const KEYS = Object.keys(GENERATED_KEYS);
+const LIVE_WHY = "A table kept current in its base's own write isn't built yet, and Mason never runs a table as another kind than its file says.";
+const isPlain = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
+const refusal = (key, message, options = [], why = null) => ({ key, message, why, options });
+/** The file's text read as a definition, or everything wrong with it. */
+export function parseGeneratedText(text, name, file) {
+    const at = { name: `${name.schema}.${name.table}`, schema: name.schema, table: name.table, file };
+    const doc = parseDocument(text, { uniqueKeys: true });
+    if (doc.errors.length)
+        return { ...at, kind: null, refusals: [refusal(null, `it is not YAML: ${doc.errors[0].message.split("\n")[0]}`)] };
+    const v = doc.toJS();
+    if (!isPlain(v))
+        return { ...at, kind: null, refusals: [refusal(null, `it is ${v === null || v === undefined ? "empty" : "not a mapping"}`, ["write kind: scheduled, target_lag: 5m and sql: SELECT …"])] };
+    const refusals = [];
+    for (const k of Object.keys(v))
+        if (!KEYS.includes(k))
+            refusals.push(refusal(k, `is not a key${didYouMean(k, KEYS)}`, [`a generated table takes ${KEYS.join(", ")}`]));
+    const kind = typeof v.kind === "string" ? v.kind : null;
+    const lagWritten = v.target_lag !== undefined;
+    const lag = lagWritten ? parseLag(v.target_lag) : null;
+    if (v.kind === undefined || v.kind === null)
+        refusals.push(refusal("kind", "is missing", ["write kind: scheduled with a target_lag, e.g. target_lag: 5m"], "Mason never decides how a table stays current for you."));
+    else if (kind === "live") {
+        refusals.push(refusal("kind", "live isn't available yet", ["set kind: scheduled with a target_lag, e.g. target_lag: 5m"], LIVE_WHY));
+        if (lagWritten)
+            refusals.push(refusal("target_lag", "contradicts kind: live", ["remove target_lag", "or set kind: scheduled"], "A live table is as fresh as its base, so it has no lag to choose."));
+    }
+    else if (kind !== "scheduled")
+        refusals.push(refusal("kind", `${String(v.kind)} is not a kind${kind === null ? "" : didYouMean(kind, ["live", "scheduled"])}`, ["write kind: scheduled with a target_lag, e.g. target_lag: 5m"]));
+    else if (!lagWritten)
+        refusals.push(refusal("target_lag", "is missing", ["write how stale is acceptable, e.g. target_lag: 5m"], "A scheduled table may lag its base, so its file says by how much."));
+    if (lag !== null && "problem" in lag)
+        refusals.push(refusal("target_lag", lag.problem));
+    const text1 = (key) => {
+        const x = v[key];
+        if (x === undefined)
+            return null;
+        if (typeof x === "string" && x.trim() !== "")
+            return x;
+        refusals.push(refusal(key, key === "order_by" ? "is one SQL string, as after ORDER BY" : "is empty or not text", key === "order_by" ? ["write order_by: day, tenant_id: a list would split on the commas inside an expression"] : []));
+        return null;
+    };
+    const sql = text1("sql");
+    if (v.sql === undefined)
+        refusals.push(refusal("sql", "is missing", ["write sql: | and the SELECT that gives the table"]));
+    const orderBy = text1("order_by"), comment = text1("comment");
+    if (refusals.length > 0 || sql === null || lag === null || "problem" in lag)
+        return { ...at, kind, refusals };
+    return { ...at, sql, orderBy: orderBy?.trim() ?? null, targetLag: lag, comment };
+}
+/** a file's name as the table it names: `requests.yml`, `reports.daily.yml`; or why it names none */
+function tableOfFile(file) {
+    const name = file.replace(/\.ya?ml$/, "");
+    const bad = tableNameProblem(name);
+    if (bad !== null)
+        return bad;
+    const dot = name.indexOf(".");
+    const schema = dot < 0 ? "public" : name.slice(0, dot), table = dot < 0 ? name : name.slice(dot + 1);
+    if (schema === "_mason" || schema === "information_schema" || schema.startsWith("pg_") || table.startsWith("pg_") || table.includes("__mason"))
+        return `${name} is a name Postgres or Mason keeps for itself`;
+    return { schema, table };
+}
+/** The folder as it is now: every definition in shape, and every file refused. A folder that is not there holds none. */
+export function readGenerated(folder, root) {
+    const dir = generatedDir(folder);
+    let names;
+    try {
+        names = fs.readdirSync(dir).filter((f) => /\.ya?ml$/.test(f) && !f.startsWith(".")).sort();
+    }
+    catch (e) {
+        if (e.code === "ENOENT")
+            return { definitions: [], refused: [] };
+        throw e;
+    }
+    const out = { definitions: [], refused: [] };
+    const seen = new Map();
+    for (const name of names) {
+        const full = path.join(dir, name), shown = shownPath(root, full);
+        const stem = name.replace(/\.ya?ml$/, "");
+        const t = tableOfFile(name);
+        if (typeof t === "string") {
+            const dot = stem.indexOf(".");
+            out.refused.push({ name: dot < 0 ? `public.${stem}` : stem, schema: dot < 0 ? "public" : stem.slice(0, dot), table: dot < 0 ? stem : stem.slice(dot + 1), file: shown, kind: null, refusals: [refusal(null, t, ["rename the file: its name is the table's"])] });
+            continue;
+        }
+        const table = `${t.schema}.${t.table}`, before = seen.get(table);
+        if (before !== undefined) {
+            out.refused.push({ name: table, ...t, file: shown, kind: null, refusals: [refusal(null, `names the table ${table}, as ${before} does`, ["keep one"])] });
+            continue;
+        }
+        seen.set(table, shown);
+        let text;
+        try {
+            text = fs.readFileSync(full, "utf8");
+        }
+        catch (e) {
+            out.refused.push({ name: table, ...t, file: shown, kind: null, refusals: [refusal(null, `it cannot be read: ${e.message}`)] });
+            continue;
+        }
+        const read = parseGeneratedText(text, t, shown);
+        if ("refusals" in read)
+            out.refused.push(read);
+        else
+            out.definitions.push(read);
+    }
+    return out;
+}

package/dist/src/cli/guides.js

@@ -7,18 +7,99 @@
  *
  * A guide's name is never a command's (tests/cli-help.test.ts): one word never means two pages.
  */
-import { DEMO_ADDED_TABLES, DEMO_TABLES, DEMO_TABLE_TO_ADD } from "../demo/seed.js";
+import { DEFAULT_MEMORY_RULE, memorySize, MEMORY_FLOOR_BYTES, OUTSIDE_RESERVE_BYTES } from "../duck/store.js";
+import { DEMO_ADDED_TABLES, DEMO_GENERATED, DEMO_TABLES, DEMO_TABLE_TO_ADD } from "../demo/seed.js";
 /** The guides, by the name `mason help <guide>` takes. */
 export const GUIDES = {
     agent: "driving Mason from a coding agent: the commands in order, --json and its shapes, exit codes, failures",
     production: "going to production: the mason-data image, its settings, Docker Compose, logical replication, upgrades",
+    generated: "generated tables: a table declared in SQL, kept current by Mason, read by name",
 };
 /** A guide's text: built at run time from the registry's and the commands' own tables. */
 export async function guideText(guide) {
     if (guide === "agent")
         return agentGuide();
+    if (guide === "generated")
+        return generatedGuide();
     return productionGuide();
 }
+/** THE GENERATED TABLES' PAGE: its keys are the reader's own table (src/cli/generated-files.ts `GENERATED_KEYS`), so it
+ *  cannot name a key the file does not take. */
+async function generatedGuide() {
+    const { GENERATED_DIR, GENERATED_KEYS } = await import("./generated-files.js");
+    const w = Math.max(...Object.keys(GENERATED_KEYS).map((k) => k.length));
+    const keys = Object.entries(GENERATED_KEYS).map(([k, what]) => `    ${k.padEnd(w)}  ${what}`).join("\n");
+    return `GENERATED TABLES — a table declared in SQL, kept current by Mason, read by name
+
+  A statement that repeats expensive work on every read can read a table that holds its answer instead. One file per
+  table in mason/${GENERATED_DIR}/, committed beside the sources file. The file's name is the table's: requests.yml is
+  public.requests, reports.daily.yml is reports.daily.
+
+    kind: scheduled
+    target_lag: 5m
+    sql: |
+      SELECT tenant_id, date_trunc('day', created_at, 'UTC') AS day, count(*) AS orders
+      FROM orders
+      GROUP BY 1, 2
+    order_by: day, tenant_id
+
+  THE KEYS
+${keys}
+
+  HOW IT STAYS CURRENT
+    A scheduled table runs its whole SQL over one snapshot, into a table beside the old one, and swaps it in. The old
+    table serves until then. It runs again once a base has changed and target_lag comes due; nothing runs over idle
+    bases. A run shares the queue that first syncs use, one at a time, and reads nothing from your database.
+    A run that fails leaves the table serving what it held, and mason status says why.
+    A table may read another generated table. Parents are built first; a base's change re-runs the whole chain above
+    it over one snapshot, and the chain swaps in at once. A table that reads itself, through others or not, is refused.
+    A parent whose run failed, or that was refused before it was ever built, holds back its children, and mason status
+    says why; a parent already built whose file is then refused keeps being re-run from what it was built from, and
+    carries its children. A table's lag is measured against the base, across the chain: the oldest change below it not
+    yet in it. So a child's target_lag can't be shorter than its parent's, and is refused if it is.
+    mason wait <table> waits until a table holds every change synced when the wait began.
+
+  WHAT IS REFUSED, WITH THE WAY OUT
+    A file with a mistake, or SQL Mason cannot store, is refused by file, key and reason. Nothing else stops, and a
+    table already built keeps serving and being re-run from what it was built with. A table has no session, so SQL that
+    reads a time zone or a setting is refused: name the zone (t AT TIME ZONE 'UTC', date_trunc('day', t, 'UTC')).
+    A daily table in one zone doesn't serve a dashboard in another.
+    A numeric column is stored exactly, with Postgres's digits, wherever its types fix the scale: one passed through,
+    its min or max, a sum (of a bigint too), and + - * of those, negated, abs, mod, round(x, n) and trunc(x, n) (n
+    below 0 too), floor, ceil, a cast to numeric(p,s) (of a JSON number's text too), and a CASE, GREATEST, LEAST or
+    COALESCE whose values have one scale, up to 18 decimals; a sign is stored too. Refused, each with its reason and
+    the way out: a division (avg is one: store it as a double, or divide where you read), a variance or stddev, which
+    Postgres computes exactly at decimals it picks per value, rounded, cast to numeric(p,s) or not (store it as a
+    double), a sqrt, exp, ln, log or power, whose decimals Postgres picks per value, rounded, cast to numeric(p,s) or
+    not (store it as a double), whatever the decimals of what it is computed from, a JSON number's text
+    cast to numeric with no scale (cast it to numeric(p,s)), more than 18 decimals, at the end or on the way
+    (round it, or give the values fewer decimals first), a sum whose decimals the rows decide (cast the values to one
+    scale first), a CASE, GREATEST, LEAST or COALESCE of values with differing decimals (cast it to numeric(p,s)), and a
+    numeric cast from a double (store the double it is). Each keeps its reason read back from its text (::text,
+    ::varchar, || '', a jsonb's ->>, a WITH column), cast to numeric(p,s) or not; only a stored column's or a JSON
+    number's text is a text's. Each is said as Postgres types the value (sqrt or avg of a
+    double is a double, so cast to numeric it is a cast from a double) wherever it sits in the column (under an
+    operator, a coalesce, a round, a cast or an aggregate, or in a scalar subquery, a WITH query, a subquery in FROM or
+    a UNION's arm), its way out your own SQL with the cast moved, the aggregate kept: sum(sqrt(x)::double precision),
+    or the WITH query, subquery or UNION the cast belongs in, given whole. A run that fails (a value
+    past 38 digits, bigint out of range, a division by zero) says Postgres's error and SQLSTATE, and is tried again.
+    An enum and a citext are stored as themselves:
+    an enum sorts in its declared order, a citext compares without case.
+    Not available yet: kind: live, and a domain or a composite type's column.
+
+  WHO READS IT
+    A role with tables: all reads it. A role with a list of tables reads it once it names it, as mason/roles.yml names
+    any table. It inherits no limits from the tables it reads: carry a tenant column and filter it in the role's where:.
+
+  SEEING IT
+    mason start               reads the folder, says what is refused, builds what is new, and keeps looking
+    mason status              each table's kind, method, state and lag
+    mason status --json       the same as generatedTables, with each refusal's key, reason and options
+    mason logs                each run: its rows and seconds, or why it failed
+
+  EXPLAIN says how a generated table is kept current on its first line, as it does for any table.
+`;
+}
 /**
  * THE AGENT'S PAGE (two readers: a person wants feedback and short, scannable responses; an agent wants clear,
  * unambiguous context and machine-readable formats; both want the next steps): what a
@@ -29,17 +110,29 @@ export async function guideText(guide) {
 async function agentGuide() {
     // imported here, not at the top: the registry imports this file for the HELP usage
     const { STEPS } = await import("./registry.js");
-    const { FAILURE_CODES } = await import("./command.js");
+    const { EXIT_CODES, FAILURE_CODES, TIMEOUT_COMMANDS } = await import("./command.js");
+    const { GENERATED_DIR } = await import("./generated-files.js");
     const w = Math.max(...STEPS.flatMap(([, cs]) => cs.map((c) => c.name.length))) + "mason  --json".length;
     const commands = STEPS.map(([step, cs]) => `  ${step}\n${cs.map((c) => `    ${`mason ${c.name}${Object.hasOwn(c.options, "json") ? " --json" : ""}`.padEnd(w)}  ${c.summary}`).join("\n")}`).join("\n");
-    const codes = Object.entries(FAILURE_CODES).map(([code, what]) => `    ${code.padEnd(12)} ${what}`).join("\n");
+    const codes = Object.entries(FAILURE_CODES).map(([code, what]) => `    ${code.padEnd(13)} ${what}`).join("\n");
     return `DRIVING MASON FROM A CODING AGENT — the same commands a person runs, answered as data
 
-  Every command prints for a person, or with --json one JSON object for a program. One prints text alone: mason help
-  (this reference); mason mcp's stdout is the MCP protocol's, so it takes no --json and says a failure to start on
-  stderr; mason logs --json is the log's entries ({ at, text }), or with --requests the request log, --events the sync's
-  events, --running the statements running now. Nothing a command prints
-  refers to a file this package does not ship.
+  Every command prints for a person, or with --json one JSON object for a program. mason mcp's stdout is the MCP
+  protocol's, so it takes no --json and says a failure to start on stderr; mason logs --json is the log's entries
+  ({ at, text }), or with --requests the request log, --events the sync's events, --running the statements running now.
+  mason version --json is { version, next }. Nothing a command prints refers to a file this package does not ship.
+
+  EVERY COMMAND'S SPEC, AS DATA: mason <command> --help --json (or mason help <command> --json) is { command: <spec>,
+  next }; mason help --json is { version, commands: [<spec>…], guides: [{ name, summary }], failureCodes: [{ code,
+  meaning }], next }. A <spec> is read from the same description --help prints, so the two cannot drift:
+    name, summary, usage   usage is the text --help prints
+    forms                  [{ synopsis, positionals }]: each form of the synopsis, and its words that are no option
+    options                [{ name, short, type, multiple, default, value }]: every option, --help too; type
+                           "string" or "boolean"; short, default and value (a string option's placeholder) null if none
+    json                   the top-level keys of its --json answer, next last; null when it takes no --json
+    needs                  "project", "run" or null
+    stdout                 "protocol" for mason mcp, else null
+    exitCodes              [{ code, meaning }]: the exit codes below it can exit with
 
   AN AGENT WITH MCP (Claude Code, Claude Desktop, any MCP client) adds Mason as an MCP server first, then asks with
   its tools rather than through a shell: tables (each synced table's rows and how current it is), describe, query (rows with
@@ -59,10 +152,8 @@ ${commands}
 
   A FAILURE is { "error": { "code", "message", "why", "fix" }, "next": […] } on stdout ("why" and "fix" where there is
   more to say than the message), and the exit code says it — the same code exits the same wherever it is found:
-    0  done
-    1  failed, with the reason
-    2  the command line is wrong (without --json the usage follows)
-    124  ran out of time: mason wait
+${Object.entries(EXIT_CODES).map(([code, meaning]) => `    ${code}  ${code === "124" ? `${meaning}: ${TIMEOUT_COMMANDS}` : meaning}`).join("\n")}
+  A wrong command line's next steps end with mason <command> --help, and its fix says what to change.
   The codes:
 ${codes}
 
@@ -116,6 +207,8 @@ ${codes}
   With no database to hand, in a new folder: mason init demo --json && mason start --json — a demo database Mason
   hosts itself (${DEMO_TABLES.join(", ")}), changed every few seconds, polled every 5 s (its sync: poll 5s). Init adds
   ${DEMO_ADDED_TABLES.join(", ")}; ${DEMO_TABLE_TO_ADD} is left to add: mason add ${DEMO_TABLE_TO_ADD} --json && mason wait --json, after mason start.
+  It also writes a generated table, ${DEMO_GENERATED.table} (mason/${GENERATED_DIR}/${DEMO_GENERATED.file}: orders and revenue per
+  day and status, over the synced orders): mason wait ${DEMO_GENERATED.table} --json, after mason start.
 `;
 }
 const REPO = "https://github.com/embeddable-hq/mason-data/blob/main";
@@ -157,20 +250,27 @@ names the tables the server syncs, and its url: names the variable that holds th
      DATABASE_URL           the database to sync from (the variable the sources file's url: names)
      MASON_API_KEY          what the users' passwords are made with (openssl rand -hex 24)
      MASON_PROJECT          your mason/ folder, mounted (default ./mason)
+     MASON_PROJECT_READONLY=1   when that folder is deployed as committed: every edit beside the server is refused
      MASON_DATA             the volume whose .mason/ holds the synced data and the state (default .)
      MASON_PG_HOST=0.0.0.0 and TLS: MASON_PG_TLS_CERT / MASON_PG_TLS_KEY, or MASON_PG_SELF_SIGNED_TLS=1, or
      MASON_PG_PLAIN=1 behind a TLS proxy of your own
    Docker Compose: ${REPO}/deploy/mason-data/README.md. One container per volume, never two: the synced data lets one process
    write its file, so run one replica, with no autoscaling, and stop the old container before the new one starts.
+   THE WEB PAGE stays on the server's loopback, so publish nothing for it: mason ui --ssh <user@server> on your machine
+   opens it through your own ssh (--mason "docker exec -i <container> mason" for a container), until Ctrl-C.
    GET /healthz on MASON_HEALTH_PORT (4140) answers 503 while it starts and 200 once it serves; the first start takes
    about 15 seconds before that, then syncs the tables. GET /metrics on the same port is for Prometheus: numbers only
    (tables by phase, lag, ages, sizes, statements), never a table's or a user's name, so it needs no secret.
-   MEMORY: the engine may use half the container's memory limit by default (else half the machine's RAM); the other
-   half is the page cache, Node and the catalog (~820 MB). Give the container enough that the page cache holds the
+   MEMORY: by default the engine may use ${DEFAULT_MEMORY_RULE};
+   the rest is the page cache, Node and the catalog (~820 MB).
+   Give it ${memorySize(MEMORY_FLOOR_BYTES)} or more (Mason itself was measured using up to 2.3 GiB beside the engine, and the kernel
+   killed it in less): under it, with MASON_DUCKDB_MEMORY unset, a start goes on and warns, in the log, mason status and
+   mason doctor, that the system may stop it. Set, it is your word at any size, and mason doctor says what it leaves. Give the container enough that the page cache holds the
    columns your dashboards read: a query over a column it does not hold runs at the disk's speed. The benchmark's synced data
    was about a fifth of its tables in Postgres (10.1 GiB to 1.8 GiB, on a 15 GiB machine); mason status says yours.
-   To set it, MASON_DUCKDB_MEMORY=4GB with the container's limit about twice that. A statement past it fails alone
-   (53200) and the run goes on; a value near the container's limit is an OOM kill.
+   To set it, MASON_DUCKDB_MEMORY=4GB with the container's limit about twice that, never under it plus ${memorySize(OUTSIDE_RESERVE_BYTES)}.
+   A statement past it fails alone (53200) and the run goes on; a value near the container's limit is an OOM kill.
+   What spills past memory takes at most half of what is free on the disk (MASON_SPILL_LIMIT sets it); a statement past that fails alone (53400).
    CPU: memory first, then cores. The benchmark's machine (4 vCPU Graviton3, 15 GiB) answered 32 clients at 10 M rows,
    12 statements a second at a p95 of 3.5 s, as many a second as at 8 clients: more clients waited longer. A first sync's
    readers take about a core each while it runs (MASON_COPY_PARALLEL, default 3). The image is arm64 (the measured