@mason-data/cli 0.7.0 → 0.8.0

package/dist/src/cli/run-server.js

@@ -12,9 +12,9 @@
  * request log (src/duck/requests.ts). Added is a fact about the project's files: a table is added when its sources file
  * names it.
  *
- * THE WEB PAGE (src/web/, `mason ui`, docs/WEB-UI.md): under `mason start` it also serves one page and its two files,
- * from an allowlist read when it starts (`pageFiles`), and the users the page manages (`/api/users`, the functions
- * `mason user` calls). The page reads the same JSON the commands read; a refusal carries `failure`, the CLI's Failure
+ * THE WEB PAGE (src/web/, `mason ui`, docs/WEB-UI.md): under `mason start` it also serves one page and its files (the
+ * shell, its pages' modules, the stylesheet), from an allowlist read when it starts (`pageFiles`), and the users the
+ * page manages (`/api/users`, the functions `mason user` calls). The page reads the same JSON the commands read; a refusal carries `failure`, the CLI's Failure
  * shape (message, why, fix, next), so the page says it as the command would. `mason serve`'s run serves no page.
  */
 import * as fs from "node:fs";
@@ -26,29 +26,45 @@ import { projectFingerprint } from "../lib/project-files.js";
 import { guard, SECURITY_HEADERS } from "./security.js";
 import { maskedUrl } from "./preflight.js";
 import { errorJson, nextJson } from "./command.js";
-import { readiness } from "./readiness.js";
-import { addedNext, addUser, listUsers, removeUser, REMOVED_NEXT } from "./user.js";
-import { postgresLink } from "./pg-url.js";
-import { catalogTable, shownPath } from "./sources.js";
+import { phaseCounts, readiness, syncTotals, tableSync } from "./readiness.js";
+import { restartsFact } from "./supervise.js";
+import { withRestarts } from "../duck/events.js";
+import { NO_ROLES } from "../duck/grants.js";
+import { addedNext, addUser, attributesToAdd, disabledNext, disableUser, enabledNext, enableUser, listUsers, removeUser, REMOVED_NEXT, roleToAdd, withLastUsed } from "./user.js";
+import { userRole } from "./roles.js";
+import { postgresFacts, postgresLink } from "./pg-url.js";
+import { catalogTable, parseSourceText, shownPath } from "./sources.js";
 import { addEntry, Refusal, removeEntries, sourceToEdit } from "./source-edit.js";
 import { PollRefused } from "../duck/sync.js";
-/** A statement answered in this many milliseconds or more is slow: what mason status lists (src/duck/requests.ts slow). */
-export const SLOW_MS = 1000;
+import { DEFAULT_SYNC, parseSync, syncText } from "../lib/table-sync.js";
+import { eventLimit, requestFilter, requestFilterText, SLOW_MS, timeOf } from "./request-filter.js";
+import { tunnelReport } from "./share-run.js";
+import { FRESH_SAMPLES } from "./metrics.js";
+/** A statement answered in this many milliseconds or more is slow (one constant: src/cli/request-filter.ts SLOW_MS, the
+ *  request log's `slow` filter's too): what mason status lists (src/duck/requests.ts slow). */
+export { SLOW_MS };
 /**
  * The run's state, in the shape mason status / wait read (`RunState`, src/cli/commands.ts): each table of the database,
  * added or not, and each added one's copy as the sync has it, so `mason wait` is ready exactly when every added table is
- * copied.
+ * copied. LEAN unless `columns` is asked: each table's columns are most of its bytes (800 tables of 50 columns, 3.4 MB
+ * with them, 0.3 MB without), and the page polls the state every 2 s for none of them — its Columns… form reads one
+ * table's on opening (`GET /api/tables/<table>/columns`, `tableColumns`); `mason status --json` asks with them
+ * (`?columns=1`), its shape unchanged. One computation either way: the columns are left out, never built apart.
  */
-export function runState(ctx, copies, stream) {
+export function runState(ctx, copies, stream, now = Date.now(), { columns = false } = {}) {
     const folder = ctx.folder();
     const sources = ctx.sources();
     const by = new Map(copies.map((c) => [c.source, c]));
     const project = folder && fs.existsSync(folder) ? projectFingerprint(folder) : null;
     const facts = ctx.facts();
+    const repl = ctx.replication();
+    const bytes = fileBytes(ctx.duck.file);
+    const disk = ctx.duck.disk();
     const tables = facts.tables.map((t) => {
-        const c = by.get(t.source);
+        const c = by.get(t.source), entry = sources.file?.tables.find((e) => e.table === t.source);
+        const { columns: cols, ...catalog } = catalogTable(t, entry);
         return {
-            ...catalogTable(t, sources.file?.tables.find((e) => e.table === t.source)),
+            ...catalog, ...(columns ? { columns: cols } : {}),
             // a table the sync has not reached is waiting for its copy (no copy yet), never "copying 0"
             // a copy in a retired layout (src/duck/sync.ts RETIRED_LAYOUT) serves until it is copied again: done, and said
             copy: c && c.phase !== "waiting" ? { phase: (c.phase === "copied" ? "done" : c.phase === "failed" ? "failed" : "copying"),
@@ -58,12 +74,13 @@ export function runState(ctx, copies, stream) {
                 ...(c.phase === "copying" && c.progress !== undefined ? { progress: c.progress } : {}),
                 ...(c.phase === "copied" && c.retired !== undefined ? { retired: c.retired } : {}) } : null,
             error: c?.phase === "failed" ? c.error : null,
-            // copied and served, but not kept current by the poll (src/duck/sync.ts POLL_ROW_LIMIT): mason status and wait say so
-            ...(c?.unsynced ? { unsynced: c.unsynced } : {}),
-            // how the run keeps the copy current NOW, as the sync derives it once (src/duck/sync.ts KeptCurrent, `keptOf`):
-            // what `unsynced` and the stream's tables say for mason status, and EXPLAIN says over the wire, as data
+            // how the run keeps the copy current NOW, as the sync derives it once (src/duck/sync.ts KeptCurrent, `keptOf`): its
+            // sync and when it was last synced, what mason status and the stream's tables say, and EXPLAIN says over the wire
             ...(c?.kept ? { kept: c.kept } : {}),
             ...(c ? { duck: { rows: c.rows, bytes: c.bytes, seconds: c.seconds, copiedAt: c.copiedAt, error: c.error } } : {}),
+            // an added table's sync: synced, syncing, failed or waiting (src/cli/readiness.ts syncStateOf), when it was last
+            // brought current, how long that took and by what, and how old that is now
+            ...(c !== undefined || entry !== undefined ? { sync: tableSync(c, now) } : {}),
         };
     });
     return {
@@ -73,13 +90,38 @@ export function runState(ctx, copies, stream) {
         // the run serves the web page (mason ui): mason status prints its URL
         ui: ctx.page,
         source: { url: maskedUrl(ctx.source.url), host: ctx.source.host, database: ctx.source.database, remote: ctx.source.remote, demo: ctx.demo === true,
-            version: facts.versionText, standby: facts.standby, changes: ctx.changes, name: sources.file?.name ?? null, file: sources.file?.file ?? null,
-            ...(ctx.publication ? { replication: { slot: ctx.slot, publication: ctx.publication.name, owned: ctx.publication.owned } } : {}) },
+            // `replication` while a table's sync is replication (the run streams), else `poll`: each table's own is its kept
+            version: facts.versionText, standby: facts.standby, changes: repl !== null ? "replication" : "poll", name: sources.file?.name ?? null, file: sources.file?.file ?? null,
+            shownFile: sources.file ? shownPath(ctx.root, sources.file.file) : null,
+            // every table's sync unless its entry says its own: the file's top `sync:`, or the default when it writes none
+            sync: { default: syncText(sources.file?.sync ?? DEFAULT_SYNC), written: sources.file?.sync != null },
+            ...(repl !== null ? { replication: { slot: repl.slot, publication: repl.publication, owned: repl.owned } } : {}) },
         // a sources file edited into a mistake while the run is up: the run copies what it read last, and says so
-        warnings: [...ctx.warnings, ...(sources.problem ? [`${sources.problem} (Mason copies the tables it read before; mason start reads the file again)`] : [])],
+        warnings: [...ctx.warnings, ...(sources.problem ? [`${sources.problem} (Mason syncs the tables it read before; mason start reads the file again)`] : [])],
         folder, project, tables,
-        engine: { name: "duckdb", file: ctx.duck.file, bytes: fileBytes(ctx.duck.file), pollSeconds: ctx.duck.pollSeconds },
+        // where a tool connects (the Postgres API's URL, the credentials' place shown, and its psql line): the web page's
+        // Connect panel, as mason status --json says it (src/cli/pg-url.ts postgresFacts)
+        postgres: postgresFacts(ctx.users.postgresUrl),
+        // the copies in all: tables added, copied, failed, their rows, the latest sync (the file's size is engine.bytes)
+        totals: syncTotals(tables.filter((t) => t.added).length, copies),
+        // the run itself: its version, since when it is up, and its supervisor's restarts (ONE reader, restartsFact, which
+        // mason status reads with Mason stopped too)
+        // keptDays: how long the request log, the events and the usage samples are kept (MASON_HISTORY_DAYS): the page
+        // offers a time range past the week only when it is kept
+        engine: { file: ctx.duck.file, bytes, keptDays: ctx.duck.requests.days,
+            version: ctx.version, startedAt: new Date(ctx.startedAt).toISOString(), uptimeSeconds: Math.max(0, Math.round((now - ctx.startedAt) / 1000)),
+            restarts: restartsFact(ctx.home, ctx.duck.restarts) },
+        // the disk under the copy: free where its file is (the sync's own measure, the one a copy is held to), the file
+        // (engine.bytes), and what the first syncs running are projected to add (copyDiskProjection) — mason status says it
+        // the sessions logged in now, by user and client (the listener's own, in memory): mason status says how many
+        sessions: ctx.listener.openSessions(),
+        disk: { path: disk.path, freeBytes: disk.freeBytes, copyBytes: bytes, projectedBytes: disk.projectedBytes, ...(disk.error !== undefined ? { error: disk.error } : {}) },
         jsonPaths: ctx.duck.jsonPaths?.() ?? [],
+        // the roles file as the listener reads it now: its roles by name, and what is wrong with it (every statement of a
+        // session with a role is refused while there is)
+        ...(ctx.roles ? { roles: rolesState(ctx.roles()) } : {}),
+        // the share open now (mason share): where, who, and what was refused there; nothing in it logs anyone in
+        share: ctx.shares?.state() ?? null,
         // replication mode: where the stream is, its lag and what it applied (mason status and wait render it, streamLines)
         ...(stream !== null ? { stream } : {}),
     };
@@ -93,27 +135,171 @@ function fileBytes(file) {
         return null;
     } // a size is a nicety of mason status: unknown, not an error
 }
-async function state(ctx) {
-    return runState(ctx, ctx.duck.copies(), await ctx.duck.stream());
+/** `RunState.roles` (src/cli/commands.ts): the file, the roles it names and what is wrong with it now */
+export const rolesState = (r) => {
+    const attributes = Object.fromEntries([...r.grants.values()].filter((g) => g.attributes !== undefined).map((g) => [g.name, Object.fromEntries(g.attributes)]));
+    const wheres = Object.fromEntries([...r.grants.values()].filter((g) => g.filters !== undefined).map((g) => [g.name,
+        [...g.filters.values()].flatMap((f) => f.where === undefined ? [] : [{ table: f.table, at: f.at, where: f.where.text }])]));
+    return { file: r.file, roles: [...r.grants.keys()], problem: r.problem, default: r.default, all: [...r.grants.values()].filter((g) => g.all).map((g) => g.name),
+        ...(Object.keys(attributes).length > 0 ? { attributes } : {}), ...(Object.keys(wheres).length > 0 ? { wheres } : {}) };
+};
+async function state(ctx, columns = false) {
+    return runState(ctx, ctx.duck.copies(), await ctx.duck.stream(), Date.now(), { columns });
+}
+/**
+ * `GET /api/tables/<table>/columns`: one table's columns as the state says them with `?columns=1` (sources.ts
+ * catalogTable: name, type, nullable, comment, references) — what the page's Columns… form ticks, read when it opens.
+ * A table the catalog lacks (dropped since the run read it) is a 404 in the failure shape.
+ */
+export function tableColumns(ctx, table) {
+    const t = ctx.facts().tables.find((x) => x.source === table);
+    if (t === undefined) {
+        const message = `the database has no table ${JSON.stringify(table)} this user can read`;
+        throw new Refusal(404, message, { message, code: "not_found", why: "it was dropped since Mason read the catalog, or never was", next: [["mason tables", "the tables it has, as this user reads them"]] });
+    }
+    return { table: t.source, columns: catalogTable(t, undefined).columns };
+}
+/**
+ * WHAT A USAGE SAMPLE READS OF THE RUN (src/duck/usage.ts): `/api/state` at that moment, never a second computation of
+ * any of it — the copy file's bytes (`engine.bytes`), `totals`, the stalest added table's `sync.ageSeconds`, the
+ * stream's `lag` and its slot's `walHeldBytes`, the bytes free under the copy (`disk.freeBytes`), the sessions open (`sessions.open`), the copies served, and
+ * the copies by phase (`phaseCounts`, the count `/healthz` says) for `GET /metrics`.
+ */
+export async function usageFacts(ctx, now = Date.now(), { refresh = true } = {}) {
+    const copies = ctx.duck.copies();
+    const s = runState(ctx, copies, await ctx.duck.stream({ refresh }), now);
+    const ages = s.tables.flatMap((t) => (t.added && t.sync?.ageSeconds != null ? [t.sync.ageSeconds] : []));
+    const slot = s.stream?.slotWal;
+    return { copyBytes: s.engine.bytes, totals: s.totals, oldestSyncSeconds: ages.length ? Math.max(...ages) : null, streamLag: s.stream?.lag ?? null,
+        copied: copies.filter((c) => c.phase === "copied").map((c) => c.source), slotWalHeldBytes: slot !== undefined && "walHeldBytes" in slot ? slot.walHeldBytes : null,
+        diskFreeBytes: s.disk.freeBytes, sessionsOpen: s.sessions.open, tables: phaseCounts(copies),
+        streamLagReadAt: s.stream !== undefined && "lsn" in s.stream.database ? s.stream.database.at : null,
+        slotWalReadAt: slot !== undefined && "walHeldBytes" in slot ? slot.at : null };
+}
+/**
+ * WHAT `GET /metrics` READS OF THE RUN (src/cli/metrics.ts, `mason serve`'s health port): a usage sample's own facts
+ * (`usageFacts`, above: no number computed a second way), the statements since the process started as the sampler counts
+ * them as they are recorded (`statementTotals`), and the last sample's p50 / p95 (`lastDurations`).
+ *
+ * A SCRAPE NEVER REACHES THE SOURCE (the review of PR 50: the endpoint takes no credential, so its rate is anyone's):
+ * the stream's position and the slot are read as cached (`refresh: false`), never asked of the database; the sampler
+ * keeps them fresh once a minute, and a number read longer ago than `FRESH_SAMPLES` samples is said as unknown
+ * (`freshSeconds`, src/cli/metrics.ts).
+ */
+export async function metricsFacts(ctx, now = Date.now()) {
+    return { usage: await usageFacts(ctx, now, { refresh: false }), statements: ctx.duck.usage.statementTotals(), durations: ctx.duck.usage.lastDurations(),
+        at: now, freshSeconds: FRESH_SAMPLES * ctx.duck.usage.sampleSeconds };
+}
+/** A `since` / `until` of a query string, read by the one time grammar (src/cli/request-filter.ts timeOf), as an ISO time. */
+function timeParam(key, text, now) {
+    const t = timeOf(key, text, now);
+    if (typeof t !== "string")
+        throw new Refusal(400, t.message, t);
+    return t;
+}
+/** why `lastUsedAt` is null in a `GET /api/users` that did not ask for it */
+const LAST_USED_NOT_ASKED = "not asked: GET /api/users?lastUsed=1 reads it from the request log";
+const DAY_S = 86_400;
+/**
+ * `GET /api/usage?since=&until=&step=`: the samples from `since` (default an hour ago) until `until` (default
+ * now), oldest first; with `step` (seconds, at least a sample's interval) re-bucketed in one query, each count summed and
+ * each level at its highest — but the p50 and p95 of a point of more than a minute are its statements', read from the
+ * request log by when each started (`durationsBy`). And each copy's own bytes as last measured, how often a sample is taken, and the days
+ * kept. `mason usage --json` is this.
+ */
+export async function usage(ctx, q, now = Date.now()) {
+    const time = (k, fallback) => {
+        const text = q.get(k);
+        if (text === null)
+            return new Date(fallback).toISOString();
+        return timeParam(k, text, now);
+    };
+    const since = time("since", now - 3_600_000), until = time("until", now);
+    if (since >= until)
+        throw new Refusal(400, `since (${since}) must be before until (${until})`);
+    const u = ctx.duck.usage;
+    const stepText = q.get("step");
+    const step = stepText === null ? null : Number(stepText);
+    if (step !== null && (!Number.isInteger(step) || step < u.sampleSeconds || step > u.days * DAY_S)) {
+        throw new Refusal(400, `step is a whole number of seconds, ${u.sampleSeconds} (a sample's interval) to ${u.days * DAY_S} (the ${u.days} days kept), not ${JSON.stringify(stepText)}`);
+    }
+    const read = await u.samples({ since, until, ...(step === null ? {} : { stepSeconds: step }) });
+    // a point of more than a minute: its p50 and p95 are its statements', from the request log (`durationsBy`), never
+    // the highest of its minutes' (a sample keeps its minute's alone)
+    const durations = step !== null && step > u.sampleSeconds ? await ctx.duck.requests.durationsBy(since, until, step) : null;
+    const samples = durations === null ? read : read.map((s) => {
+        const d = durations.get(Date.parse(s.at));
+        return { ...s, durationP50Ms: d?.p50 ?? null, durationP95Ms: d?.p95 ?? null };
+    });
+    return { since, until, stepSeconds: step, sampleSeconds: u.sampleSeconds, keptDays: u.days, samples, copySizes: await u.copySizes() };
+}
+/**
+ * `GET /api/activity?since=`: what the Postgres API answered from `since` on (default an hour ago; the one time grammar,
+ * src/cli/request-filter.ts `timeOf`), for the Usage page's time range — the window's statements (src/duck/requests.ts
+ * `window`), who sent them (`clients`: a row per user and client, the 50 with the most statements; `clientsTotal`:
+ * how many clients and sessions in all) and the slowest that completed (`slowest`); and the
+ * days the request log keeps. Read once per range chosen, every 5 s while the range is Now.
+ */
+export async function activity(ctx, q, now = Date.now()) {
+    const sinceText = q.get("since");
+    const since = sinceText === null ? new Date(now - 3_600_000).toISOString() : timeParam("since", sinceText, now);
+    const r = ctx.duck.requests;
+    const [window, who, slowest] = await Promise.all([r.window(since, now), r.clients(since), r.slowest(since)]);
+    return { since, window, clients: who.clients, clientsTotal: who.total, slowest, keptDays: r.days };
+}
+/** how many registry entries `/api/unsupported` answers unless `limit` says, and at most */
+export const DEFAULT_UNSUPPORTED_LIMIT = 10;
+export const MAX_UNSUPPORTED_LIMIT = 100;
+/**
+ * `GET /api/unsupported?since=&limit=`: the statements refused from `since` on (default a day ago; read by the one time
+ * grammar, src/cli/request-filter.ts `timeOf`) grouped by the registry entry that refused them (src/duck/requests.ts
+ * `unsupported`), the most refused first, `limit` (1 to 100, default 10) of them; and the days the request log keeps.
+ * `mason status --json` `unsupported` is this.
+ */
+export async function unsupported(ctx, q, now = Date.now()) {
+    const sinceText = q.get("since");
+    const since = sinceText === null ? new Date(now - DAY_S * 1000).toISOString() : timeParam("since", sinceText, now);
+    const limitText = q.get("limit");
+    const limit = limitText === null ? DEFAULT_UNSUPPORTED_LIMIT : Number(limitText);
+    if (!Number.isInteger(limit) || limit < 1 || limit > MAX_UNSUPPORTED_LIMIT) {
+        throw new Refusal(400, `limit is a whole number of entries of docs/SQL-SUPPORT.md, 1 to ${MAX_UNSUPPORTED_LIMIT}, not ${JSON.stringify(limitText)}`);
+    }
+    return { since, unsupported: await ctx.duck.requests.unsupported({ since, limit }), keptDays: ctx.duck.requests.days };
+}
+/**
+ * `GET /api/events?limit=&since=&failed=1`: the newest `limit` (1 to 10000, default 40) of the sync's events merged with
+ * the run's restarts, newest last, from `since` on (`since` and `until` are read everywhere by one grammar, src/cli/request-filter.ts `timeOf`: an age, or an ISO 8601 time with its zone), the failures only with `failed=1`; and the days
+ * they are kept. `mason logs --events --json` is this; `mason status --json` `recentErrors` its latest failures.
+ */
+export async function events(ctx, q) {
+    const limit = eventLimit(q.get("limit") ?? undefined, "limit");
+    if (typeof limit !== "number")
+        throw new Refusal(400, limit.message, limit);
+    const sinceText = q.get("since");
+    const since = sinceText === null ? undefined : timeParam("since", sinceText, Date.now());
+    const o = { limit, failed: q.get("failed") === "1", ...(since === undefined ? {} : { since }) };
+    const found = await ctx.duck.events.recent(o);
+    return { events: withRestarts(found, ctx.duck.restarts(), { ...o, days: ctx.duck.events.days }), keptDays: ctx.duck.events.days };
 }
 /** The sources file as it is on disk now (src/cli/source-edit.ts sourceToEdit) — and the database it names NOW must be
  *  the one this run copies: its catalog is what an edit is checked against. */
 function sourceOnDisk(ctx) {
     const at = sourceToEdit(ctx.folder(), ctx.root, ctx.readSource);
     if (at.read.resolved !== ctx.runUrl) {
-        throw new Refusal(409, `${shownPath(ctx.root, at.file)}: url: now names ${maskedUrl(at.read.resolved)} (${at.read.from}), and this run copies ${maskedUrl(ctx.runUrl)}: ` +
-            "mason stop && mason start copies from the new one, then add or remove the table");
+        throw new Refusal(409, `${shownPath(ctx.root, at.file)}: url: now names ${maskedUrl(at.read.resolved)} (${at.read.from}), and this run syncs ${maskedUrl(ctx.runUrl)}: ` +
+            "mason stop && mason start syncs from the new one, then add or remove the table");
     }
     return at;
 }
 /** The database this run copies, as an edit is checked against it: the catalog it last read, and how it reads changes. */
-const catalogOf = (ctx) => ({ tables: ctx.facts().tables, database: ctx.source.database, replication: ctx.changes === "replication" });
+const catalogOf = (ctx) => ({ tables: ctx.facts().tables, database: ctx.source.database,
+    ...(ctx.duck.measured ? { measured: ctx.duck.measured } : {}) });
 /**
  * ADD A TABLE: the check-and-write every entrance shares (src/cli/source-edit.ts addEntry), against the file on disk and
  * this run's catalog; the caller tells the run the file changed, and the sync starts the copy.
  */
-export function pick(ctx, table, columns = ["*"], key = null) {
-    return addEntry(sourceOnDisk(ctx), catalogOf(ctx), table, columns, key);
+export function pick(ctx, table, columns = ["*"], key = null, o = {}, at = sourceOnDisk(ctx)) {
+    return addEntry(at, catalogOf(ctx), table, columns, key, o);
 }
 /** REMOVE TABLES: the check-and-write every entrance shares (src/cli/source-edit.ts removeEntries); the run then drops their copies. */
 export function removeTables(ctx, names) {
@@ -129,11 +315,49 @@ const statusOf = (f) => f.code === "usage" ? 400 : f.code === "not_found" ? 404
 export function sendFailure(res, status, f, extra = {}) {
     sendJson(res, status, { error: f.message, ...extra, failure: { ...errorJson(f), next: nextJson(f.next) } });
 }
+/**
+ * A VENDORED FILE AT A PINNED VERSION (`/vendor/<name>-<x.y.z>/…`, src/web/vendor/README.md): its bytes are that
+ * release's for good — a new version is a new folder — so the browser keeps it a year without asking again
+ * (`immutable`): a reload re-reads the page's own modules (no-store, as every answer) but not the fonts, uPlot and
+ * the icons — measured with the demo: a reload moved 222 KB where the first load moved 520 KB.
+ */
+const PINNED_VENDOR = /^\/vendor\/[a-z][a-z0-9-]*-\d+\.\d+\.\d+\//;
+const IMMUTABLE = "private, max-age=31536000, immutable";
 /** THE WEB PAGE'S ROUTES, AND THE FILE EACH IS: the only files the server ever reads for a URL. */
 export const PAGE_ROUTES = {
     "/": ["index.html", "text/html; charset=utf-8"],
     "/app.js": ["app.js", "text/javascript; charset=utf-8"],
     "/app.css": ["app.css", "text/css; charset=utf-8"],
+    // the shell's shared helpers and one module per page (src/web/app.js has the page module contract)
+    "/shared.js": ["shared.js", "text/javascript; charset=utf-8"],
+    "/pages/status.js": ["pages/status.js", "text/javascript; charset=utf-8"],
+    "/pages/tables.js": ["pages/tables.js", "text/javascript; charset=utf-8"],
+    "/pages/queries.js": ["pages/queries.js", "text/javascript; charset=utf-8"],
+    "/pages/users.js": ["pages/users.js", "text/javascript; charset=utf-8"],
+    "/pages/usage.js": ["pages/usage.js", "text/javascript; charset=utf-8"],
+    "/pages/syncs.js": ["pages/syncs.js", "text/javascript; charset=utf-8"],
+    // the charts (src/web/charts.js over uPlot) and what the page vendors whole, pinned, with its licence
+    // (src/web/vendor/README.md): served from here, never fetched from elsewhere — a box without internet reads it all
+    "/charts.js": ["charts.js", "text/javascript; charset=utf-8"],
+    "/vendor/README.md": ["vendor/README.md", "text/plain; charset=utf-8"],
+    "/vendor/geist-1.7.2/Geist-Variable.woff2": ["vendor/geist-1.7.2/Geist-Variable.woff2", "font/woff2"],
+    "/vendor/geist-1.7.2/GeistMono-Variable.woff2": ["vendor/geist-1.7.2/GeistMono-Variable.woff2", "font/woff2"],
+    "/vendor/geist-1.7.2/LICENSE.txt": ["vendor/geist-1.7.2/LICENSE.txt", "text/plain; charset=utf-8"],
+    "/vendor/zilla-slab-5.3.0/zilla-slab-latin-700-normal.woff2": ["vendor/zilla-slab-5.3.0/zilla-slab-latin-700-normal.woff2", "font/woff2"],
+    "/vendor/zilla-slab-5.3.0/LICENSE.txt": ["vendor/zilla-slab-5.3.0/LICENSE.txt", "text/plain; charset=utf-8"],
+    "/vendor/uplot-1.6.32/uPlot.esm.js": ["vendor/uplot-1.6.32/uPlot.esm.js", "text/javascript; charset=utf-8"],
+    "/vendor/uplot-1.6.32/uPlot.min.css": ["vendor/uplot-1.6.32/uPlot.min.css", "text/css; charset=utf-8"],
+    "/vendor/uplot-1.6.32/LICENSE.txt": ["vendor/uplot-1.6.32/LICENSE.txt", "text/plain; charset=utf-8"],
+    // the nav's icons (src/web/shared.js ICONS): Lucide's, one ES module per icon, each its SVG as a string
+    "/vendor/lucide-1.56.0/chart-column.mjs": ["vendor/lucide-1.56.0/chart-column.mjs", "text/javascript; charset=utf-8"],
+    "/vendor/lucide-1.56.0/layout-dashboard.mjs": ["vendor/lucide-1.56.0/layout-dashboard.mjs", "text/javascript; charset=utf-8"],
+    "/vendor/lucide-1.56.0/logs.mjs": ["vendor/lucide-1.56.0/logs.mjs", "text/javascript; charset=utf-8"],
+    "/vendor/lucide-1.56.0/menu.mjs": ["vendor/lucide-1.56.0/menu.mjs", "text/javascript; charset=utf-8"],
+    "/vendor/lucide-1.56.0/square-terminal.mjs": ["vendor/lucide-1.56.0/square-terminal.mjs", "text/javascript; charset=utf-8"],
+    "/vendor/lucide-1.56.0/table.mjs": ["vendor/lucide-1.56.0/table.mjs", "text/javascript; charset=utf-8"],
+    "/vendor/lucide-1.56.0/users.mjs": ["vendor/lucide-1.56.0/users.mjs", "text/javascript; charset=utf-8"],
+    "/vendor/lucide-1.56.0/x.mjs": ["vendor/lucide-1.56.0/x.mjs", "text/javascript; charset=utf-8"],
+    "/vendor/lucide-1.56.0/LICENSE.txt": ["vendor/lucide-1.56.0/LICENSE.txt", "text/plain; charset=utf-8"],
 };
 /**
  * The web page's files, READ ONCE when the server starts (src/web/ beside src/cli/, in a checkout and in the package
@@ -150,11 +374,70 @@ export function pageFiles(dir = fileURLToPath(new URL("../web/", import.meta.url
         catch (e) {
             throw new Error(`the web page's ${file} cannot be read (${at}: ${e.message}): this Mason is installed incompletely — npm i -g @mason-data/cli again`);
         }
-        return [route, { body, type }];
+        return [route, { body, type, immutable: PINNED_VENDOR.test(route) }];
     }));
 }
+/** How long `/api/queries`' summary is answered from the last read: the Overview asks every 30 s, Usage and Queries
+ *  too, and `mason status` at each run — each a 7-day scan of the request log (~0.15 s at 30 M rows). */
+export const SUMMARY_TTL_MS = 30_000;
+/**
+ * ONE READ SHARED FOR `ms`, by time only (`summaryCache`): every ask within `ms` of a read gets that read — the one in
+ * flight too, so two pages asking at once cost one scan. A read that fails is not kept: the next ask reads again. So the
+ * summary is up to `ms` old (docs/OBSERVABILITY.md says it).
+ */
+export function timedCache(read, ms, clock = Date.now) {
+    let kept = null;
+    return () => {
+        const now = clock();
+        if (kept !== null && now - kept.at < ms)
+            return kept.value;
+        const value = read();
+        const mine = { at: now, value };
+        kept = mine;
+        // a failure is the caller's answer (it awaits `value`); here it only means the read is not kept
+        value.catch(() => { if (kept === mine)
+            kept = null; });
+        return value;
+    };
+}
+/**
+ * `POST /api/share` { secret, role? }: a share opened, its first line of JSON, and THE ANSWER LEFT OPEN: the connection is
+ * the share (src/cli/share-run.ts), so a command gone by any means ends it. Only the password's hash arrives here. Past
+ * its first line the answer says one thing, why the run ended it.
+ */
+async function openShare(ctx, req, res) {
+    if (ctx.shares === undefined) {
+        sendFailure(res, 409, { message: "this run is on a server, and has an address of its own: mason share is for a run on a laptop", code: "refused", next: [["mason help production", "reaching a server's Postgres API"]] });
+        return;
+    }
+    const shares = ctx.shares;
+    const body = JSON.parse(await readBody(req, 64 * 1024));
+    if (typeof body.secret !== "string" || !/^[0-9a-f]{64}$/.test(body.secret))
+        throw new Refusal(400, "send { secret, role? }: the password's SHA-256 in hex");
+    if (body.role !== undefined && typeof body.role !== "string")
+        throw new Refusal(400, "role is a role's name, or left out");
+    let gone = false;
+    res.once("close", () => { gone = true; });
+    const opened = await shares.open({ secret: body.secret, role: body.role });
+    if ("failure" in opened) {
+        sendFailure(res, statusOf(opened.failure), opened.failure, opened.choices ? { choices: opened.choices } : {});
+        return;
+    }
+    const end = (why) => { void shares.close(opened.user, why).catch((e) => ctx.log(`closing the share ${opened.user} failed: ${e.message}`)); };
+    // the command went while the socket opened
+    if (gone) {
+        end("mason share went away");
+        return;
+    }
+    res.once("close", () => end("mason share went away"));
+    shares.onEnd(opened.user, (why) => { res.end(`${JSON.stringify({ ended: why })}\n`); });
+    res.writeHead(200, { "Content-Type": "application/x-ndjson" });
+    res.write(`${JSON.stringify(opened)}\n`);
+}
 export function createRunServer(ctx) {
     const saidErrors = new Set();
+    // the last day's and week's statements, read at most once every SUMMARY_TTL_MS for every page and command that asks
+    const summary = timedCache(() => ctx.duck.requests.summary(), SUMMARY_TTL_MS);
     // read once, here: nothing a request names is ever read from disk (mason serve's run serves no page)
     const page = ctx.page ? pageFiles() : new Map();
     return http.createServer((req, res) => {
@@ -174,14 +457,27 @@ export function createRunServer(ctx) {
             try {
                 const file = req.method === "GET" || req.method === "HEAD" ? page.get(url.pathname) : undefined;
                 if (file !== undefined) {
-                    res.writeHead(200, { "Content-Type": file.type, "Content-Length": file.body.length }).end(req.method === "HEAD" ? undefined : file.body);
+                    res.writeHead(200, { "Content-Type": file.type, "Content-Length": file.body.length, ...(file.immutable ? { "Cache-Control": IMMUTABLE } : {}) })
+                        .end(req.method === "HEAD" ? undefined : file.body);
                 }
                 else if (req.method === "GET" && url.pathname === "/api/ping") {
                     // what proves a run is up (src/cli/runs.ts answers): this token answered, by this process
                     sendJson(res, 200, { pid: process.pid });
                 }
                 else if (req.method === "GET" && url.pathname === "/api/state") {
-                    sendJson(res, 200, await state(ctx));
+                    // lean, as the page polls it; ?columns=1 with each table's columns (mason status --json)
+                    sendJson(res, 200, await state(ctx, url.searchParams.get("columns") === "1"));
+                }
+                else if (req.method === "GET" && /^\/api\/tables\/[^/]+\/columns$/.test(url.pathname)) {
+                    // one table's columns, read when the page's Columns… form opens (never with the state, polled every 2 s)
+                    let table;
+                    try {
+                        table = decodeURIComponent(url.pathname.split("/")[3]);
+                    }
+                    catch {
+                        throw new Refusal(400, `${url.pathname} does not name a table: GET /api/tables/<schema.table, URL-encoded>/columns`);
+                    }
+                    sendJson(res, 200, tableColumns(ctx, table));
                 }
                 else if (req.method === "GET" && url.pathname === "/api/slow") {
                     // the last day's slow statements, from the request log (src/duck/requests.ts)
@@ -190,22 +486,100 @@ export function createRunServer(ctx) {
                 else if (req.method === "GET" && url.pathname === "/api/refused") {
                     sendJson(res, 200, await ctx.duck.requests.refused());
                 }
+                else if (req.method === "GET" && url.pathname === "/api/unsupported") {
+                    // the refused statements grouped by the unsupported SQL they used (src/duck/gaps.ts): mason status
+                    sendJson(res, 200, await unsupported(ctx, url.searchParams));
+                }
                 else if (req.method === "GET" && url.pathname === "/api/requests") {
-                    const limit = Number(url.searchParams.get("limit") ?? 40);
-                    if (!Number.isInteger(limit) || limit < 1 || limit > 10_000)
-                        throw new Refusal(400, "limit is a whole number of requests, 1 to 10000");
-                    sendJson(res, 200, { requests: await ctx.duck.requests.recent({ limit, failed: url.searchParams.get("failed") === "1" }), keptDays: ctx.duck.requests.days });
+                    // the filters, read as mason logs --requests reads its options (src/cli/request-filter.ts)
+                    const q = requestFilter(requestFilterText(url.searchParams));
+                    if ("message" in q) {
+                        sendFailure(res, 400, q);
+                        return;
+                    }
+                    sendJson(res, 200, { ...await ctx.duck.requests.page(q), keptDays: ctx.duck.requests.days });
+                }
+                else if (req.method === "GET" && url.pathname === "/api/queries") {
+                    // the last day's and week's statements in one scan of the request log (mason status); asked on its own, never
+                    // with /api/state, which the page asks every 2 s
+                    // (cached SUMMARY_TTL_MS: up to 30 s old)
+                    sendJson(res, 200, { queries: await summary() });
+                }
+                else if (req.method === "GET" && url.pathname === "/api/activity") {
+                    // one time range's statements, clients and slowest (the Usage page's), from the request log
+                    sendJson(res, 200, await activity(ctx, url.searchParams));
+                }
+                else if (req.method === "GET" && url.pathname === "/api/running") {
+                    sendJson(res, 200, { running: ctx.listener.running() });
+                }
+                else if (req.method === "POST" && url.pathname === "/api/cancel") {
+                    const body = JSON.parse(await readBody(req, 64 * 1024));
+                    const whole = (x) => typeof x === "number" && Number.isInteger(x) && x >= 1;
+                    if (!whole(body.session))
+                        throw new Refusal(400, "send { session }: the session's number, as mason logs --running lists it");
+                    if (body.statement !== undefined && !whole(body.statement))
+                        throw new Refusal(400, "statement is the statement's number, as mason logs --running lists it, or left out");
+                    const r = ctx.listener.cancel(body.session, body.statement);
+                    if ("cancelled" in r) {
+                        sendJson(res, 200, r);
+                        return;
+                    }
+                    const running = ["mason logs --running", "the statements running now, with their sessions"];
+                    if ("other" in r) {
+                        sendFailure(res, 409, { message: `session ${r.other.session} runs another statement now (statement ${r.other.statement}), not statement ${r.asked}: nothing cancelled`, code: "refused",
+                            why: `statement ${r.asked} ended, and the session began another`, fix: `mason cancel ${r.other.session} --statement ${r.other.statement} stops the one it runs now`, next: [running] });
+                        return;
+                    }
+                    if ("unknown" in r) {
+                        sendFailure(res, 404, { message: `no session ${r.unknown} is open`, code: "not_found", why: "it ended, or never was", next: [running] });
+                        return;
+                    }
+                    sendFailure(res, 409, { message: `session ${r.idle} is running no statement: nothing to cancel`, code: "refused", why: "its statement ended, or it is between two", next: [running] });
+                }
+                else if (req.method === "GET" && url.pathname === "/api/logs") {
+                    // the run's last log entries from memory (mason serve has no file to read): { at, text } each, oldest first
+                    const lines = Number(url.searchParams.get("lines") ?? Math.min(40, ctx.logs.capacity));
+                    if (!Number.isInteger(lines) || lines < 1 || lines > ctx.logs.capacity)
+                        throw new Refusal(400, `lines is a whole number of log entries, 1 to ${ctx.logs.capacity} (all the run keeps)`);
+                    const sinceText = url.searchParams.get("since");
+                    const since = sinceText === null ? undefined : timeParam("since", sinceText, Date.now());
+                    sendJson(res, 200, { lines: ctx.logs.recent({ lines, ...(since !== undefined ? { since } : {}) }), kept: ctx.logs.capacity });
+                }
+                else if (req.method === "GET" && url.pathname === "/api/events") {
+                    // the sync's events and the run's restarts, newest last (src/duck/events.ts): mason logs --events, mason status
+                    sendJson(res, 200, await events(ctx, url.searchParams));
+                }
+                else if (req.method === "GET" && url.pathname === "/api/usage") {
+                    // usage over time, a sample a minute (src/duck/usage.ts): mason usage
+                    sendJson(res, 200, await usage(ctx, url.searchParams));
                 }
                 else if (req.method === "POST" && url.pathname === "/api/pick") {
                     const body = JSON.parse(await readBody(req, 64 * 1024));
                     if (typeof body.table !== "string" || typeof body.picked !== "boolean")
-                        throw new Refusal(400, "send { table, picked, columns?, key? }");
+                        throw new Refusal(400, "send { table, picked, columns?, key?, sync?, keep? }");
+                    // a sync by the sources file's own grammar (src/lib/table-sync.ts), as mason add --sync reads it
+                    const sync = body.sync === undefined ? undefined : parseSync(body.sync);
+                    if (sync !== undefined && "problem" in sync)
+                        throw new Refusal(400, `sync ${JSON.stringify(body.sync)}: ${sync.problem}`);
                     if (body.columns !== undefined && !(Array.isArray(body.columns) && body.columns.every((c) => typeof c === "string")))
                         throw new Refusal(400, "columns is a list of column names, or [\"*\"]");
                     if (body.key !== undefined && !(Array.isArray(body.key) && body.key.length > 0 && body.key.every((c) => typeof c === "string")))
                         throw new Refusal(400, "key is a list of column names");
                     if (body.picked) {
-                        const r = pick(ctx, body.table, body.columns ?? ["*"], body.key ?? null);
+                        // the file as it was (its text, read once with the edit): an edit that sets a table to sync: replication the
+                        // database cannot serve is undone below
+                        const at = sourceOnDisk(ctx);
+                        const r = pick(ctx, body.table, body.columns ?? ["*"], body.key ?? null, { ...(sync !== undefined ? { sync } : {}), keep: body.keep === true }, at);
+                        // A LIVE SWITCH INTO REPLICATION is held to the start's checks (wal_level, the role, a free slot): one the
+                        // database cannot serve would leave the table, polled until now, NOT kept current for ever — the file is put
+                        // back as it was, byte for byte, and the refusal says what the database lacks and the line that polls it
+                        const now = r.already || ctx.checkReplication === undefined ? null : parseSourceText(fs.readFileSync(at.file, "utf8"), at.parsed.name, at.file);
+                        const refused = now === null || "problems" in now ? null : await ctx.checkReplication(now);
+                        if (refused !== null) {
+                            fs.writeFileSync(at.file, at.text);
+                            sendFailure(res, statusOf(refused), refused);
+                            return;
+                        }
                         if (!r.already)
                             ctx.log(`added ${r.table} to ${r.file}`);
                         ctx.sourcesChanged();
@@ -219,21 +593,58 @@ export function createRunServer(ctx) {
                         sendJson(res, 200, { ...r, state: await state(ctx) });
                     }
                 }
+                else if (req.method === "POST" && url.pathname === "/api/share") {
+                    await openShare(ctx, req, res);
+                }
+                else if (req.method === "POST" && /^\/api\/share\/[^/]+$/.test(url.pathname)) {
+                    // what the command knows of its tunnel, as it knows it: mason status says it, and the run stops the client
+                    const body = tunnelReport(JSON.parse(await readBody(req, 64 * 1024)));
+                    if (body === null)
+                        throw new Refusal(400, "send { via?, host?, port?, endsAt?, state?, pid? }: what the command knows of its tunnel");
+                    const user = decodeURIComponent(url.pathname.slice("/api/share/".length));
+                    if (ctx.shares?.update(user, body) !== true) {
+                        sendFailure(res, 404, { message: `no share ${user} is open`, code: "not_found" });
+                        return;
+                    }
+                    sendJson(res, 200, { ok: true });
+                }
                 else if (req.method === "GET" && url.pathname === "/api/users") {
-                    // mason user list --json's shape: the users, and whether each connects
-                    const r = listUsers(ctx.users.file, ctx.users.apiKey);
+                    // mason user list --json's shape: the users, and whether each connects (and with ?lastUsed=1 when each was last used)
+                    const r = listUsers(ctx.users.file, ctx.users.apiKey, postgresLink(ctx.users.postgresUrl), ctx.roles?.() ?? NO_ROLES);
                     if ("message" in r) {
                         sendFailure(res, statusOf(r), r);
                         return;
                     }
-                    sendJson(res, 200, { ...r, next: nextJson([["mason user add <name>", "a user for a tool: its password and URL, printed once"]]) });
+                    // the last use is a scan of the request log (~0.15 s on a week of 50 statements a second): only when asked
+                    // (?lastUsed=1: mason user list, the Users page every 30 s), never for the Overview's poll, which does not show it
+                    const used = url.searchParams.get("lastUsed") === "1" ? await withLastUsed(r.users, ctx.duck.requests, ctx.log)
+                        : { users: r.users, lastUsedNote: LAST_USED_NOT_ASKED, requestLogDays: ctx.duck.requests.days };
+                    sendJson(res, 200, { ...used, urlNote: null, next: nextJson([["mason user add <name>", "a user for a tool: its password and URL, printed once"]]) });
                 }
                 else if (req.method === "POST" && url.pathname === "/api/users") {
                     const body = JSON.parse(await readBody(req, 64 * 1024));
                     if (typeof body.name !== "string")
                         throw new Refusal(400, "send { name }: the user's name");
+                    if (body.role !== undefined && typeof body.role !== "string")
+                        throw new Refusal(400, "send { name, role }: the role's name, a string");
+                    if (body.set !== undefined && !(Array.isArray(body.set) && body.set.every((x) => typeof x === "string")))
+                        throw new Refusal(400, "send { name, role, set }: set is a list of \"attribute=value\" strings, as mason user add --set takes them");
+                    // the role must be one the roles file names NOW (src/cli/user.ts roleToAdd): refused before anything is written
+                    const roleProblem = body.role === undefined ? null : roleToAdd(body.role, ctx.roles?.() ?? NO_ROLES);
+                    if (roleProblem !== null) {
+                        sendFailure(res, statusOf(roleProblem), roleProblem);
+                        return;
+                    }
+                    // its attributes, held to the role as mason user add holds them (src/cli/user.ts attributesToAdd): one grammar
+                    const rolesNow = ctx.roles?.() ?? NO_ROLES;
+                    const attributes = attributesToAdd(body.name, typeof body.role === "string" ? body.role : userRole({}, rolesNow).role, body.set ?? [], rolesNow);
+                    if ("refused" in attributes) {
+                        sendFailure(res, statusOf(attributes.refused), attributes.refused);
+                        return;
+                    }
                     // mason user add --json's shape: its password, once, and the URL to paste (this run is up, so it has one)
-                    const r = addUser(ctx.users.file, ctx.users.apiKey, body.name, { link: postgresLink(ctx.users.postgresUrl), database: ctx.users.database });
+                    const r = addUser(ctx.users.file, ctx.users.apiKey, body.name, { link: postgresLink(ctx.users.postgresUrl), database: ctx.users.database, roles: ctx.roles?.() ?? NO_ROLES,
+                        ...(typeof body.role === "string" ? { role: body.role } : {}), attributes: attributes.values });
                     if ("message" in r) {
                         sendFailure(res, statusOf(r), r);
                         return;
@@ -242,6 +653,37 @@ export function createRunServer(ctx) {
                     ctx.log(`user ${added.user} added${replaced ? " (in place of one the API key's rotation had removed)" : ""}`);
                     sendJson(res, 200, { ...added, next: nextJson(addedNext(added.user, true)) });
                 }
+                else if (req.method === "POST" && /^\/api\/users\/[^/]+\/(disable|enable)$/.test(url.pathname)) {
+                    // mason user disable|enable --json's shape, through its own functions: kept, its password unchanged
+                    const [, , , encoded, verb] = url.pathname.split("/");
+                    let name;
+                    try {
+                        name = decodeURIComponent(encoded);
+                    }
+                    catch {
+                        throw new Refusal(400, `${url.pathname} does not name a user: POST /api/users/<name>/${verb}`);
+                    }
+                    if (verb === "disable") {
+                        const r = disableUser(ctx.users.file, name);
+                        if ("message" in r) {
+                            sendFailure(res, statusOf(r), r);
+                            return;
+                        }
+                        if (!r.already)
+                            ctx.log(`user ${r.disabled.name} disabled`);
+                        sendJson(res, 200, { ...r, next: nextJson(disabledNext(r.disabled.name)) });
+                    }
+                    else {
+                        const r = enableUser(ctx.users.file, name);
+                        if ("message" in r) {
+                            sendFailure(res, statusOf(r), r);
+                            return;
+                        }
+                        if (!r.already)
+                            ctx.log(`user ${r.enabled.name} enabled`);
+                        sendJson(res, 200, { ...r, next: nextJson(enabledNext(r.enabled.name)) });
+                    }
+                }
                 else if (req.method === "DELETE" && url.pathname.startsWith("/api/users/")) {
                     let name;
                     try {