@mason-data/cli 0.7.0 → 0.8.0

package/dist/src/cli/replication-mode.js

@@ -1,15 +1,17 @@
 /**
- * `mason start --replication` (the journey to production, item 6): a project's run reading its database's
- * change log as production does — a publication, a replication slot, `wal_level = logical` — instead of polling it with
- * SELECTs (src/cdc/poll.ts). A developer then meets the setup production needs on their own database first. The mode is
- * the project's (`.mason/config.json` `changes`), kept between starts until `--poll` takes it back.
+ * LOGICAL REPLICATION FOR THE TABLES WHOSE `sync:` SAYS SO (the journey to production, item 6; issue 33): a project's run
+ * reading its database's change log as production does — a publication, a replication slot, `wal_level = logical` —
+ * for each table whose sync is `replication`, beside the polled ones (src/cdc/poll.ts). A developer then meets the setup
+ * production needs on their own database first. The sources file decides it, per table, and nothing else does
+ * (`mason start --replication`, `--poll`, `--publication` and `MASON_CHANGES` are gone: src/lib/table-sync.ts
+ * `removedStartFlags`, `changesRetired`).
  *
  * What it costs the developer's database, said where it happens: Mason creates its slot there (`mason_<deployment>`), and
- * a slot holds WAL while nobody reads it — so `mason stop` says so, and `mason destroy`, or going back to polling, drops
+ * a slot holds WAL while nobody reads it — so `mason stop` says so, and `mason destroy`, or no table replicated any more, drops
  * it (`dropOwnReplication`), verified gone, by the name the start that made it RECORDED (`madeReplication`), never one
  * computed again. The PUBLICATION is Mason's own (`mason_<deployment>`, made and dropped with
- * the slot) unless the operator names one (`--publication <name>`, else `PG_PUBLICATION`, else the one
- * an earlier start recorded): that one is the database owner's, read as it is and never created, altered or dropped
+ * the slot) unless the operator names one (`publication:` at the top of the sources file, else `PG_PUBLICATION`):
+ * that one is the database owner's, read as it is and never created, altered or dropped
  * (src/duck/publication.ts) — on a read replica it is the only way, since CREATE PUBLICATION is refused on a standby.
  *
  * In the ClickHouse service this was cut from, the two modes kept row versions that could not continue each other (a
@@ -20,6 +22,8 @@
 import { createHash } from "node:crypto";
 import { replicationNames } from "../cdc/source.js";
 import { connectSource, systemIdentifier } from "./preflight.js";
+import { STATE_DIR } from "./project.js";
+import { replicatedEntries, silentOverSlot, syncWritten } from "./sources.js";
 import { readServerFacts, readSlotFacts, replicationChecks } from "./replication-checks.js";
 /** Postgres's longest identifier (NAMEDATALEN - 1): a longer slot name is refused, a longer publication's cut with a notice. */
 export const PG_NAME_BYTES = 63;
@@ -38,8 +42,8 @@ export function deploymentOf(project, source) {
     return `${full.slice(0, room - 9)}_${createHash("sha256").update(full).digest("hex").slice(0, 8)}`;
 }
 /**
- * The publication and slot a project's run reads: the publication `named` (the flag, `PG_PUBLICATION` or the recorded
- * one: `namedPublication`), else Mason's own — by the change reader's own rule (src/cdc/source.ts `replicationNames`),
+ * The publication and slot a project's run reads: the publication `named` (the sources file's or `PG_PUBLICATION`:
+ * `namedPublication`, or the recorded one), else Mason's own — by the change reader's own rule (src/cdc/source.ts `replicationNames`),
  * which also says whether it is Mason's to make. The slot is always the project's own.
  */
 export function projectReplication(name, source, named = null) {
@@ -48,27 +52,69 @@ export function projectReplication(name, source, named = null) {
 }
 /**
  * WHAT REPLICATION MODE MADE ON THE DATABASE, as the start that made it recorded it (`config.json` `replication`): what
- * `--poll` and `mason destroy` drop. Recorded, never computed again, so a name that changes since — the sources file
+ * a start whose sources file replicates no table, and `mason destroy`, drop. Recorded, never computed again, so a name that changes since — the sources file
  * renamed, the naming rule — never leaves a slot that nobody drops, holding the database's WAL for ever. A project with
  * no record (none started in replication mode since it was kept) has the names the rule gives now.
  */
 export function madeReplication(cfg, source) {
     return cfg.replication ?? projectReplication(cfg.name, source, cfg.publication ?? null);
 }
-/** The publication an operator named, and where: `--publication`, else `PG_PUBLICATION`, else the one an earlier start
- *  of this copy recorded (`config.json` `publication`), else none (Mason's own). */
-export function namedPublication(flag, env, recorded) {
-    if (flag)
-        return { name: flag, from: "flag" };
+/**
+ * THE PUBLICATION A START READS, and where it is named: the sources file's `publication:` (its `${PG_PUBLICATION}`
+ * resolved), else `PG_PUBLICATION` in the environment, else none — Mason's own, which `publication: null` asks for past
+ * the environment too. NEVER the one an earlier start recorded, read silently: a record the file does not take up is
+ * refused (`recordRefusal`), so what runs is what is committed.
+ */
+export function namedPublication(file, env) {
+    if (file.publicationName !== null)
+        return { name: file.publicationName, from: "file" };
+    if (file.publication && "own" in file.publication)
+        return null;
     if (env.PG_PUBLICATION)
         return { name: env.PG_PUBLICATION, from: "env" };
-    if (recorded)
-        return { name: recorded, from: "recorded" };
     return null;
 }
-/** The mode a start runs: the flag, else the project's, else polling. */
-export function changesOf(flag, recorded) {
-    return flag ?? recorded ?? "poll";
+/**
+ * WHAT `.mason/config.json` RECORDS AND THE SOURCES FILE DOES NOT SAY, refused before anything is written or connected
+ * (one report, for the start that is typed, the run it spawns and `mason serve`). The file says how a project replicates,
+ * and a file that is silent is an omission, not a decision, whose price is on someone's production database:
+ *   - a project that streams (a slot recorded, of any release, or an older Mason's `changes: "replication"`) under a file
+ *     naming tables with no `sync:` at its top or on any entry would have its slot dropped, and streaming again copies
+ *     every table whole. Any `sync:` written is the file saying so: `off` or `poll <interval>` confirms the drop,
+ *     `replication` keeps the slot. The run's re-read holds the same line (`silentOverSlot`, `rereadSources`).
+ *   - a project that replicates through a publication it recorded, with no `publication:` and no `PG_PUBLICATION`,
+ *     would silently make and read a publication of Mason's own. The literal `publication: null` asks for that on purpose.
+ *   - `publication: null` beside a set `PG_PUBLICATION` says two things, and one of them is a CREATE on the source.
+ * `file` null: there is no sources file (a serve would write one saying `sync: off`, which must not read as a choice). A
+ * project is told the first two at once: the first edit must not meet the second refusal.
+ */
+export function recordRefusal(cfg, file, env, shown, cmd) {
+    const recorded = cfg.publication ?? null;
+    const declined = file?.publication != null && "own" in file.publication;
+    const unnamed = recorded !== null && !declined && namedPublication(file ?? { publicationName: null }, env) === null;
+    const next = [[cmd, "once the file says it; nothing was dropped or changed"]];
+    if (declined && env.PG_PUBLICATION) {
+        return { message: `${shown} says publication: null, and PG_PUBLICATION names ${env.PG_PUBLICATION}`, code: "refused",
+            why: "one asks Mason to make a publication of its own on your database, the other to read that one, and the file would win by saying nothing of the setting",
+            fix: `for Mason's own, unset PG_PUBLICATION; to read ${env.PG_PUBLICATION}, take publication: null out of ${shown}`, next };
+    }
+    const slotRecorded = (cfg.replication ?? null) !== null;
+    const older = cfg.changes === "replication";
+    if (file === null ? slotRecorded || older : older ? !syncWritten(file) : silentOverSlot(slotRecorded, file)) {
+        const slot = cfg.replication?.slot;
+        const made = `${slot === undefined ? "its replication slot" : `replication slot ${slot}`}${(cfg.replication?.owned ?? recorded === null) ? " and Mason's publication" : ""}`;
+        return { message: `this project streams, and ${file === null ? "it has no sources file to say so" : `${shown} does not say so: no sync: at its top or on an entry`}`, code: "refused",
+            why: `${older ? `an older Mason recorded that this project streams every table (changes: replication in ${STATE_DIR}/config.json)` : `${STATE_DIR}/config.json records that this project streams`}; left unsaid, sync is off, which drops ${made}, and streaming again reads every table whole from your database`,
+            fix: file === null
+                ? `mount the project folder with its sources file (MASON_PROJECT), or make ${shown} there with kind: postgres, url: \${DATABASE_URL}, tables: and the tables it streams, and sync: replication${unnamed ? ` and publication: ${recorded}` : ""} to keep streaming; sync: off or sync: poll <interval> in it instead confirms dropping ${made}`
+                : `to keep streaming, write sync: replication${unnamed ? ` and publication: ${recorded}` : ""} at the top of ${shown}; to drop ${made}, confirm it with sync: off or sync: poll <interval> there`, next };
+    }
+    if (unnamed && file !== null && replicatedEntries(file).length > 0) {
+        return { message: `this project reads publication ${recorded}, and ${shown} does not name it`, code: "refused",
+            why: `${STATE_DIR}/config.json records ${recorded}, its owner's, named at an earlier start; with no publication: line and no PG_PUBLICATION, Mason would make and read a publication of its own on your database instead`,
+            fix: `to keep reading it, write publication: ${recorded} at the top of ${shown}; for a publication of Mason's own, write publication: null there`, next };
+    }
+    return null;
 }
 /**
  * Whether the developer's database can be read by replication: what refuses it (wal_level, the role — REPLICATION, a
@@ -81,7 +127,7 @@ export function readinessVerdict(facts, slot, slotFacts, repl) {
     // a standby takes no CREATE PUBLICATION (it is read-only): the publication must be the primary's, named
     if (facts.standby && repl.owned) {
         checks.push({ name: "publication", ok: false, detail: `this database is a read replica, where Mason cannot make its publication ${repl.publication} (CREATE ` +
-                "PUBLICATION is refused on a standby). Create one on the primary for the tables you add, then name it: mason start --replication --publication <name>" });
+                "PUBLICATION is refused on a standby). Create one on the primary for the tables whose sync: is replication, then name it: publication: <name> at the top of the sources file" });
     }
     // a laptop's database has no cap on its slots by default: worth saying, never a reason to refuse a dev run
     return { refused: checks.filter((x) => x.ok === false && x.name !== "WAL cap"), warnings: checks.filter((x) => (x.name === "WAL cap" && x.ok === false) || x.warn === true) };
@@ -97,18 +143,51 @@ export async function replicationReadiness(source, repl) {
         c.pg.close();
     }
 }
+/**
+ * A TABLE SET TO `sync: replication` WHILE THE RUN IS UP, in a run that replicated nothing when it started (so the
+ * database was never checked for it): the start's own checks, as the edit's refusal — what the database lacks, and the
+ * line that keeps the table polled — or null when it may replicate. `tables` are the entries' names (`orders`).
+ */
+export function liveReadinessFailure(ready, tables, shown, database) {
+    const refused = "failure" in ready ? [{ name: "database", ok: false, detail: ready.failure }] : ready.refused;
+    if (!refused.length)
+        return null;
+    const one = tables.length === 1;
+    return { message: `${tables.join(", ")} cannot be kept current by logical replication from ${database}: ${refused.map((c) => c.name).join(", ")}`, code: "refused",
+        why: refused.map((c) => `${c.name}: ${c.detail}${c.sql ? ` (${c.sql})` : ""}`).join("\n"),
+        fix: `poll ${one ? "it" : "them"} instead: mason add ${tables[0]} --sync "poll 5m"${one ? "" : " (and each of the others)"}, or sync: poll 5m on ${one ? "its entry" : "their entries"} in ${shown}, ` +
+            "which needs none of this — or fix the database as said, then mason stop && mason start",
+        next: [[`mason add ${tables[0]} --sync "poll 5m"`, "polls it with SELECTs instead"]] };
+}
 /** The statements that remove what replication mode made, for a person to run when Mason could not: the slot, and the
  *  publication only when it is Mason's own (a named one is its owner's). */
 export function dropStatements(o) {
     return `SELECT pg_drop_replication_slot('${o.slot}') FROM pg_replication_slots WHERE slot_name = '${o.slot}' AND NOT active;` +
         (o.owned ? ` DROP PUBLICATION IF EXISTS ${o.publication};` : "");
 }
+/**
+ * WHAT THE RECORD NAMES THAT THIS START NO LONGER READS, to drop before the record names the new one (each holds what
+ * Mason made on a database nothing would drop otherwise): a slot under another name (the sources file renamed) goes
+ * with Mason's own publication; Mason's own publication alone when the project now reads another (one named), on the
+ * same slot — a review finding, the own one was left on the database after the switch and after mason destroy.
+ */
+export function staleReplication(was, now) {
+    if (was === null)
+        return null;
+    if (was.slot !== now.slot)
+        return { drop: was, what: "both", why: `this source's slot is ${now.slot} now` };
+    if (was.owned && (was.publication !== now.publication || !now.owned)) {
+        return { drop: was, what: "publication", why: `the replicated tables are read through publication ${now.publication} now${now.owned ? "" : ", its owner's"}` };
+    }
+    return null;
+}
 /**
  * Drop what replication mode made on the developer's database — the slot, and the publication when it is Mason's own,
  * never one the operator named — and verify it is gone: what was dropped, in words. A slot a reader still holds is never
- * dropped (the run must be stopped first): that is a failure.
+ * dropped (the run must be stopped first): that is a failure. `what: "publication"` drops Mason's own publication alone
+ * (the slot is still read).
  */
-export async function dropOwnReplication(source, o, from = null) {
+export async function dropOwnReplication(source, o, from = null, what = "both") {
     const c = await connectSource(source);
     if ("failure" in c)
         throw new Error(`could not connect to drop them: ${c.failure.message}`);
@@ -119,11 +198,11 @@ export async function dropOwnReplication(source, o, from = null) {
         if (from?.systemIdentifier) {
             const now = await systemIdentifier(pg);
             if (now !== null && now !== from.systemIdentifier) {
-                throw new Error(`${source.database} on ${source.host} is another server than the copy came from (system identifier ${now}, the copy's ${from.systemIdentifier}), where the slot is: put that server's URL back in the sources file and run mason destroy again, or drop them there by hand`);
+                throw new Error(`${source.database} on ${source.host} is another server than the synced data came from (system identifier ${now}, the synced data's ${from.systemIdentifier}), where the slot is: put that server's URL back in the sources file and run mason destroy again, or drop them there by hand`);
             }
         }
         const dropped = [];
-        const slot = await readSlotFacts(pg, o.slot);
+        const slot = what === "both" ? await readSlotFacts(pg, o.slot) : null;
         if (slot) {
             if (slot.active)
                 throw new Error(`replication slot ${o.slot} is in use by a reader: stop it first`);
@@ -136,7 +215,7 @@ export async function dropOwnReplication(source, o, from = null) {
             dropped.push(`publication ${o.publication}`);
         }
         // verified, never assumed: a slot left behind holds the database's WAL for ever
-        if (await readSlotFacts(pg, o.slot))
+        if (what === "both" && await readSlotFacts(pg, o.slot))
             throw new Error(`replication slot ${o.slot} is still there after pg_drop_replication_slot`);
         if (o.owned && await hasPub())
             throw new Error(`publication ${o.publication} is still there after DROP PUBLICATION`);

package/dist/src/cli/request-filter.js

@@ -0,0 +1,134 @@
+import { OUTCOMES } from "../api/request-log.js";
+import { parseRequestCursor, requestCursor } from "../duck/requests.js";
+import { didYouMean } from "../lib/did-you-mean.js";
+/** How many requests a read returns unless it says. */
+export const DEFAULT_REQUEST_LIMIT = 40;
+export const MAX_REQUEST_LIMIT = 10_000;
+/** A statement answered in this many milliseconds or more is slow: what mason status lists, and what the `slow` filter
+ *  keeps (`--slow`, `slow=1`). One constant, here, where the filter is read; src/cli/run-server.ts re-exports it. */
+export const SLOW_MS = 1000;
+/** The longest text `search` takes: a fragment of a statement, never a statement pasted whole. */
+export const MAX_SEARCH = 200;
+/** How many sync events a read returns unless it says, and the most it may. */
+export const DEFAULT_EVENT_LIMIT = 40;
+export const MAX_EVENT_LIMIT = MAX_REQUEST_LIMIT;
+const AGE = /^(\d+)(s|m|h|d)$/;
+const AGE_MS = { s: 1000, m: 60_000, h: 3_600_000, d: 86_400_000 };
+/** a date, or a date and time with its zone (a time without one would be read in this machine's zone: refused) */
+const ISO = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2}(\.\d{1,6})?)?(Z|[+-]\d{2}:?\d{2}))?$/;
+const usage = (message, fix) => ({ message, code: "usage", ...(fix !== undefined ? { fix } : {}) });
+export const QUERY_NAMES = (k) => k;
+/**
+ * THE TIME GRAMMAR, ONE FOR EVERY ENTRANCE (`--since` / `--until` of mason logs and mason usage, and the run's `since` and
+ * `until` of /api/requests, /api/events, /api/usage and /api/logs): an ISO 8601 date or time with its zone, or an age
+ * before `now` (30s, 30m, 2h, 7d), as an ISO time; a time with no zone is refused, as is anything else.
+ */
+export function timeOf(name, text, now) {
+    const age = AGE.exec(text.trim());
+    if (age !== null)
+        return new Date(now - Number(age[1]) * AGE_MS[age[2]]).toISOString();
+    const t = ISO.test(text.trim()) ? Date.parse(text.trim()) : NaN;
+    if (Number.isNaN(t)) {
+        return usage(`${name} ${JSON.stringify(text)} is not a time`, "an age (30m, 2h, 7d) or an ISO 8601 date or time with its zone (2026-10-28, 2026-10-28T09:00:00Z)");
+    }
+    return new Date(t).toISOString();
+}
+/** `-n` / `limit` of the sync's events: a whole number, 1 to the cap, said in the entrance's own spelling of the key. */
+export function eventLimit(text, key) {
+    const limit = text === undefined ? DEFAULT_EVENT_LIMIT : Number(text);
+    if (!Number.isInteger(limit) || limit < 1 || limit > MAX_EVENT_LIMIT)
+        return usage(`${key} is a whole number of events, 1 to ${MAX_EVENT_LIMIT}`);
+    return limit;
+}
+/**
+ * The filters read, or the first thing wrong with them, its key spelled as the entrance spells it (`name`). `failed`
+ * is every outcome but ok, so it is never named beside `outcome`; `slow` keeps the ones that took SLOW_MS or more, and
+ * `search` the statements whose text has it (any case, at most MAX_SEARCH characters); `since` must come before `until`; `before` is a
+ * cursor a page handed on, passed back as it came.
+ */
+export function requestFilter(f, now = Date.now(), name = QUERY_NAMES) {
+    const limit = f.limit === undefined ? DEFAULT_REQUEST_LIMIT : Number(f.limit);
+    if (!Number.isInteger(limit) || limit < 1 || limit > MAX_REQUEST_LIMIT)
+        return usage(`${name("limit")} is a whole number of requests, 1 to ${MAX_REQUEST_LIMIT}`);
+    const q = { limit };
+    if (f.outcome !== undefined) {
+        if (f.failed === true) {
+            return usage(`${name("failed")} is every outcome but ok: name it or ${name("outcome")}, never both`, `${name("outcome")} refused,failed,cancelled is ${name("failed")}`);
+        }
+        const names = f.outcome.split(",").map((o) => o.trim()).filter((o) => o !== "");
+        if (!names.length)
+            return usage(`${name("outcome")} names one or more of ${OUTCOMES.join(", ")}`);
+        for (const o of names) {
+            if (!OUTCOMES.includes(o))
+                return usage(`${name("outcome")} ${JSON.stringify(o)} is none of ${OUTCOMES.join(", ")}${didYouMean(o, OUTCOMES)}`);
+        }
+        q.outcomes = [...new Set(names)];
+    }
+    if (f.failed === true)
+        q.failed = true;
+    for (const k of ["since", "until"]) {
+        const text = f[k];
+        if (text === undefined)
+            continue;
+        const t = timeOf(name(k), text, now);
+        if (typeof t !== "string")
+            return t;
+        q[k] = t;
+    }
+    if (q.since !== undefined && q.until !== undefined && q.since >= q.until) {
+        return usage(`${name("since")} (${q.since}) is not before ${name("until")} (${q.until}): no request started in between`);
+    }
+    for (const k of ["user", "client"]) {
+        const text = f[k];
+        if (text === undefined)
+            continue;
+        if (text === "")
+            return usage(`${name(k)} names ${k === "user" ? "a mason user" : "a client's application_name"}: it is empty`);
+        q[k] = text;
+    }
+    if (f.slow === true)
+        q.minMs = SLOW_MS;
+    if (f.search !== undefined) {
+        if (f.search.trim() === "")
+            return usage(`${name("search")} is a piece of a statement's text: it is empty`);
+        if (f.search.length > MAX_SEARCH)
+            return usage(`${name("search")} is a piece of a statement's text, at most ${MAX_SEARCH} characters: it has ${f.search.length}`);
+        q.search = f.search;
+    }
+    if (f.before !== undefined) {
+        const c = parseRequestCursor(f.before);
+        if (c === null)
+            return usage(`${name("before")} ${JSON.stringify(f.before)} is not a position in the request log`, `pass back the before a page answered with, as it came`);
+        q.before = c;
+    }
+    return q;
+}
+/** The query string the run reads a RequestQuery from (requestFilter again, on its side): its times as resolved. */
+export function requestFilterParams(q) {
+    const p = new URLSearchParams({ limit: String(q.limit) });
+    if (q.failed === true)
+        p.set("failed", "1");
+    if (q.outcomes !== undefined)
+        p.set("outcome", q.outcomes.join(","));
+    if (q.since !== undefined)
+        p.set("since", q.since);
+    if (q.until !== undefined)
+        p.set("until", q.until);
+    if (q.user !== undefined)
+        p.set("user", q.user);
+    if (q.client !== undefined)
+        p.set("client", q.client);
+    if (q.minMs !== undefined)
+        p.set("slow", "1");
+    if (q.search !== undefined)
+        p.set("search", q.search);
+    if (q.before !== undefined)
+        p.set("before", requestCursor(q.before.at, q.before.seq));
+    return p;
+}
+/** A query string's filters as text (`GET /api/requests`): what requestFilter reads. */
+export function requestFilterText(p) {
+    const get = (k) => p.get(k) ?? undefined;
+    return { limit: get("limit"), outcome: get("outcome"), failed: p.get("failed") === "1", since: get("since"), until: get("until"),
+        user: get("user"), client: get("client"), slow: p.get("slow") === "1", search: get("search"), before: get("before") };
+}