@mason-data/cli 0.7.0 → 0.8.0

package/dist/src/cli/runs.js

@@ -61,7 +61,7 @@ export const END_GRACE_MS = 15_000;
 export function notAnswering(p, run) {
     return { problem: `Mason for ${p.config.name} is running (pid ${run.pid}) but does not answer its commands (${run.control.url}, 3 s)`, code: "not_answering",
         why: "a statement may be holding its main thread",
-        next: [["mason stop", `end it (SIGTERM, then SIGKILL after ${END_GRACE_MS / 1000} s); its copy stays and the next mason start resumes`], ["mason logs", "what it was doing"]] };
+        next: [["mason stop", `end it (SIGTERM, then SIGKILL after ${END_GRACE_MS / 1000} s); its synced data stays and the next mason start resumes`], ["mason logs", "what it was doing"]] };
 }
 /** Whether `pid` is a Mason run, by its command line (`mason start … --attach`, src/cli/background.ts). ps exits 1 for a pid that is gone. */
 export function masonProcess(pid) {

package/dist/src/cli/security.js

@@ -22,19 +22,21 @@
  * no Origin: a browser sends an Authorization header across sites only after a CORS preflight, which these servers
  * never approve, so a page on another site cannot send one at all.
  *
- * Cookies are per host, not per port, so one cookie serves every page of a run; a page of another run, or one
- * the token does not match, is refused. Pure over the request's headers, so the refusals are unit-tested
+ * A browser keeps cookies per host, never per port, so the cookie is NAMED for the run's port (`cookieName`,
+ * `mason_dev_<port>`): two runs on one machine each keep their own, and opening one's page never logs the other's out
+ * (one name for every run did). A page of another run, or one the token does not match, is refused. Pure over the request's headers, so the refusals are unit-tested
  * (tests/cli-security.test.ts).
  */
 import { randomBytes, timingSafeEqual } from "node:crypto";
-export const COOKIE = "mason_dev";
+/** The cookie of the run whose server is on `port`: one per run, as a browser shares a host's cookies across ports. */
+export const cookieName = (port) => `mason_dev_${port}`;
 export function newToken() { return randomBytes(32).toString("hex"); }
 /** Constant-time string equality (a token compared byte by byte leaks its prefix through timing). */
 export function sameSecret(a, b) {
     const x = Buffer.from(a), y = Buffer.from(b);
     return x.length === y.length && timingSafeEqual(x, y);
 }
-export function cookieOf(header, name = COOKIE) {
+export function cookieOf(header, name) {
     for (const part of (header ?? "").split(";")) {
         const eq = part.indexOf("=");
         if (eq > 0 && part.slice(0, eq).trim() === name)
@@ -42,8 +44,8 @@ export function cookieOf(header, name = COOKIE) {
     }
     return null;
 }
-export function setCookieHeader(token) {
-    return `${COOKIE}=${token}; HttpOnly; SameSite=Strict; Path=/`;
+export function setCookieHeader(token, port) {
+    return `${cookieName(port)}=${token}; HttpOnly; SameSite=Strict; Path=/`;
 }
 /** Whether `req` may be served by the server on `port` whose run token is `token`. */
 export function guard(req, port, token) {
@@ -62,9 +64,9 @@ export function guard(req, port, token) {
         if (!sameSecret(offered, token))
             return { ok: false, status: 403, reason: "the token in the URL is not this run's" };
         url.searchParams.delete("token");
-        return { ok: false, status: 303, location: url.pathname + (url.search || "") + url.hash, setCookie: setCookieHeader(token) };
+        return { ok: false, status: 303, location: url.pathname + (url.search || "") + url.hash, setCookie: setCookieHeader(token, port) };
     }
-    const cookie = cookieOf(req.cookie);
+    const cookie = cookieOf(req.cookie, cookieName(port));
     if (cookie === null || !sameSecret(cookie, token)) {
         return { ok: false, status: 403, reason: "this is mason start's command server for the mason command: a request carries the run's token (Authorization: Bearer, from .mason/config.json)" };
     }

package/dist/src/cli/serve.js

@@ -13,6 +13,8 @@
  *     platform restarts it; there is no supervisor of its own (src/cli/supervise.ts is `mason start`'s).
  *   - A HEALTH ENDPOINT (`healthBody`): `GET /healthz` on MASON_HEALTH_PORT, 503 while it starts and stops, 200 once it
  *     answers. It needs no credential, so it names nothing of the database: counts only.
+ *   - METRICS BESIDE IT (src/cli/metrics.ts): `GET /metrics` on the same port, Prometheus's text format, numbers only
+ *     (counts, lag, ages, sizes) for the same reason: never a table's, a user's or a host's name, never a statement.
  *   - A CLEAN STOP ON SIGTERM: the run's own stop (a copy in flight interrupted and rolled back, DuckDB closed).
  *
  * The commands beside it work as on a laptop, in the data folder: `docker compose exec mason mason add orders`.
@@ -24,9 +26,17 @@ import { failed, parseCommand } from "./command.js";
 import { dev } from "./dev.js";
 import { parseSource } from "./preflight.js";
 import { NAME_RE, newConfig, readProject, STATE_DIR, writeConfig } from "./project.js";
-import { newSourceText, OLD_TABLES_DIR, parseSourceText, readSource, sourceFileNames, sourceNameFor, sourcesDir } from "./sources.js";
+import { newSourceText, OLD_TABLES_DIR, parseSourceText, readSource, sourceFileNames, sourceNameFor, sourcesDir, sourcesTop, writtenDatabase } from "./sources.js";
+import { connectDatabase, DATABASE_SETTING, databaseNameProblem } from "../lib/database-name.js";
 import { appEnvValue } from "../lib/dotenv.js";
-import { version } from "./version.js";
+import { imagePinProblem, PIN_FILE } from "../lib/project-version.js";
+import { commandPinProblem } from "./pin.js";
+import { recordRefusal } from "./replication-mode.js";
+import { changesRetiredText, pollMsRetiredText } from "../lib/table-sync.js";
+import { historyDaysSetting, historyDaysWrongText } from "../duck/history.js";
+import { runningVersion, version } from "./version.js";
+import { phaseCounts } from "./readiness.js";
+import { METRICS_CONTENT_TYPE, metricsText } from "./metrics.js";
 export const SERVE = {
     name: "serve",
     summary: "run Mason on a server or in a container, in the foreground, configured by the environment",
@@ -34,32 +44,38 @@ export const SERVE = {
 
   Runs Mason as mason start --attach does, for a server or a container: nothing asked, its settings from the
   environment, its state in one folder (a volume), its log on stdout, GET /healthz for the platform's health check,
-  and a clean stop on SIGTERM. The tables it copies are the ones the project folder's sources file names
-  (sources/<name>.yml): make it with mason init and mason add on a laptop and commit it, or with mason add beside the
-  running server. A project folder with no sources file gets one, url: \${DATABASE_URL}, with no table yet.
+  GET /metrics for Prometheus (numbers only: no name, no statement), and a clean stop on SIGTERM. The tables it
+  syncs are the ones the project folder's sources file names (sources/<name>.yml): make it with mason init and mason
+  add on a laptop and commit it, or with mason add beside the running server. A project folder with no sources file
+  gets one, url: \${DATABASE_URL}, with no table yet.
 
     DATABASE_URL=postgres://mason:…@db.internal:5432/app MASON_API_KEY=… mason serve
     docker compose exec mason mason add public.orders     beside it, in its container
 
   Settings (the environment):
-    DATABASE_URL            the database to copy from, as the sources file's url: \${DATABASE_URL} names it (the variable
+    DATABASE_URL            the database to sync from, as the sources file's url: \${DATABASE_URL} names it (the variable
                             the file names is required); read-only: SELECTs, or a replication slot
+    PG_PUBLICATION          the publication the tables under sync: replication are read through, the database owner's,
+                            never altered: the sources file's publication: first (publication: \${PG_PUBLICATION} reads
+                            this one), else this; default none: Mason makes its own
     MASON_API_KEY           what the users' passwords are made with (mason user add), at least 32 characters (required)
     MASON_PROJECT           the project folder: sources/, mason.yml (default ./mason)
-    MASON_DATA              the folder whose .mason/ holds the copy and the state: the volume (default .)
-    MASON_NAME              the project's name, the database name clients connect to (default mason)
-    MASON_CHANGES           poll (SELECTs, the default) or replication (the change log; PG_PUBLICATION names one)
-    MASON_HEALTH_PORT       /healthz's port (default 4140), on MASON_HEALTH_HOST (default 0.0.0.0)
+    MASON_DATA              the folder whose .mason/ holds the synced data and the state: the volume (default .)
+    MASON_NAME              the project's name, which the replication slot is named for (default mason); renaming it
+                            over the same synced data is refused
+    MASON_PG_DATABASE       the database name a tool connects with, for a sources file with no database: (default mason;
+                            the two must agree when both are set)
+    MASON_HEALTH_PORT       /healthz's and /metrics' port (default 4140), on MASON_HEALTH_HOST (default 0.0.0.0)
     MASON_PG_HOST, MASON_PG_PORT   the Postgres API's address (default 127.0.0.1:5433)
     MASON_PG_TLS_CERT, MASON_PG_TLS_KEY   its certificate, required off loopback; or MASON_PG_SELF_SIGNED_TLS=1
                             for one made in .mason/, or MASON_PG_PLAIN=1 behind a TLS proxy of your own
-    MASON_DUCKDB_MEMORY     the copy's memory, 4GB or 512MiB (default half the container's memory limit, else half the
+    MASON_DUCKDB_MEMORY     the engine's memory, 4GB or 512MiB (default half the container's memory limit, else half the
                             machine's RAM; mason doctor says it); a statement past it fails alone with 53200
-    MASON_POLL_MS           how often the poll reads the tables' changes, in ms (default 5000)
     MASON_STATEMENT_TIMEOUT   each statement's limit, as Postgres's statement_timeout (default 5min; 0 for none)
-    MASON_REQUEST_LOG_DAYS  how long the request log keeps a statement, in days (default 7)
+    MASON_HISTORY_DAYS      how many days Mason keeps its history: the request log, the sync's events, the usage
+                            samples and the synced tables' sizes (default 7)
     MASON_STORE_JSON_PATHS  on (the default) stores the JSON paths statements keep reading as columns; off stores none
-    MASON_COPY_PARALLEL     a copy's chunks read at once, each a backend on the database (1 to 8, default 3)
+    MASON_COPY_PARALLEL     a first sync's chunks read at once, each a backend on the database (1 to 8, default 3)
     MASON_COPY_CHUNK_SECONDS   how long each chunk takes to read, at the rate measured (1 to 300, default 12)
 
   --json   a refusal to start as data: { error, next }; once it serves, its output is its log
@@ -85,9 +101,21 @@ export function serveSettings(env, cwd) {
     const name = env.MASON_NAME || "mason";
     if (!NAME_RE.test(name))
         wrong.push(`MASON_NAME=${JSON.stringify(name)} is not a project name: lower case letters, digits, - and _, at most 48`);
-    const changes = env.MASON_CHANGES || null;
-    if (changes !== null && changes !== "poll" && changes !== "replication")
-        wrong.push(`MASON_CHANGES=${JSON.stringify(changes)}: poll or replication`);
+    // the database name a tool connects with: the sources file's, else this, and never two names at once
+    wrong.push(...databaseSettingProblems(project, env));
+    // how each table is kept current, and how often a table is polled, are the sources file's sync: now, per table
+    // (src/lib/table-sync.ts): the old settings refused HERE with the others, before anything is written or connected
+    const top = () => sourcesTop(path.dirname(project), project);
+    const changes = changesRetiredText(env, top);
+    if (changes !== null)
+        wrong.push(changes);
+    const pollMs = pollMsRetiredText(env, top);
+    if (pollMs !== null)
+        wrong.push(pollMs);
+    // the history's days: MASON_HISTORY_DAYS, a number of days; the old name refused (src/duck/history.ts)
+    const days = historyDaysSetting(env);
+    if (typeof days !== "number")
+        wrong.push(historyDaysWrongText(days));
     const selfSigned = env.MASON_PG_SELF_SIGNED_TLS || "0";
     if (selfSigned !== "0" && selfSigned !== "1")
         wrong.push(`MASON_PG_SELF_SIGNED_TLS=${JSON.stringify(selfSigned)}: 1, or 0 (the default)`);
@@ -102,11 +130,27 @@ export function serveSettings(env, cwd) {
             next: [["mason serve --help", "every setting, with its default"]] };
     }
     return {
-        name, apiKey, changes: changes, selfSignedTls: selfSigned === "1",
+        name, apiKey, selfSignedTls: selfSigned === "1",
         project, data,
         health: { host: env.MASON_HEALTH_HOST || "0.0.0.0", port },
     };
 }
+/**
+ * THE DATABASE NAME AS A SETTING (src/lib/database-name.ts): a name, and one the sources file does not contradict, since
+ * two names for one thing would have to be chosen between. A file that cannot be read is serveSource's to say.
+ */
+function databaseSettingProblems(project, env) {
+    const set = env[DATABASE_SETTING];
+    if (!set)
+        return [];
+    const bad = databaseNameProblem(set);
+    if (bad !== null)
+        return [`${DATABASE_SETTING}: ${bad}`];
+    const written = writtenDatabase(project, path.dirname(project));
+    if ("message" in written || written.database === null || written.database === set)
+        return [];
+    return [`${DATABASE_SETTING}=${JSON.stringify(set)}, and the sources file says database: ${written.database}: keep one`];
+}
 /**
  * THE DATABASE'S URL AS A SETTING (the review of the sources file: an unset URL was found only after the other
  * settings passed, two deploys for two variables): the variable the project's sources file names — DATABASE_URL when it
@@ -141,25 +185,25 @@ function urlSettingProblems(project, data, env) {
         return [];
     const found = appEnvValue(data, variable, env, { shellFirst: true });
     if (found === null)
-        return [`${variable} is not set: the database to copy from, as the sources file's url: \${${variable}} names it`];
+        return [`${variable} is not set: the database to sync from, as the sources file's url: \${${variable}} names it`];
     const p = parseSource(found.value);
     return "problem" in p ? [`${variable}: ${p.problem}`] : [];
 }
 /**
  * THE CONFIG A SERVE RUNS ON, written from its settings at every start, over what the runs before recorded (the server
  * the copy was taken from, the publication, the certificate): the environment is the operator's, the rest the run's. A
- * project `mason init` made is a developer's and is never taken over; a name that moved would be another slot and
- * another database name over the same copy, so it is refused.
+ * project `mason init` made is a developer's and is never taken over; a name that moved would be another slot over the
+ * same copy, so it is refused.
  */
 export function serveConfig(s, existing) {
     const dir = path.join(s.data, STATE_DIR);
     if (existing === null) {
         const config = { ...newConfig(s.name, s.project, true), apiKey: s.apiKey, served: true };
-        return { config, said: [`a new copy in ${dir}`] };
+        return { config, said: [`new synced data in ${dir}`] };
     }
     if ("broken" in existing) {
         return { message: `the state in ${dir} cannot be read: ${existing.file}: ${existing.broken}`, code: "bad_config",
-            fix: `MASON_DATA names a folder of its own for mason serve; move ${dir} aside to copy again` };
+            fix: `MASON_DATA names a folder of its own for mason serve; move ${dir} aside to sync again` };
     }
     const was = existing.config;
     if (was.served !== true) {
@@ -168,17 +212,27 @@ export function serveConfig(s, existing) {
             fix: "point MASON_DATA at an empty folder (the volume), or run mason start here" };
     }
     if (was.name !== s.name) {
-        return { message: `the copy in ${dir} is the project ${was.name}, and MASON_NAME says ${s.name}`, code: "refused",
-            why: "the name is the database name clients connect to and names the replication slot: another name over the same copy is another project",
-            fix: `set MASON_NAME=${was.name} again, or give the new name a new MASON_DATA (an empty volume), which copies again` };
+        return { message: `the synced data in ${dir} is the project ${was.name}, and MASON_NAME says ${s.name}`, code: "refused",
+            why: "the name names the replication slot: another name over the same synced data is another project (the database name a tool connects with is not this: MASON_PG_DATABASE, or database: in the sources file)",
+            fix: `set MASON_NAME=${was.name} again, or give the new name a new MASON_DATA (an empty volume), which syncs again` };
     }
-    const said = [`the copy in ${dir}, resumed`];
+    const said = [`the synced data in ${dir}, resumed`];
     if (was.apiKey !== s.apiKey)
         said.push("MASON_API_KEY changed: every mason user is removed (mason user add each again)");
     // the source is the sources file's; a database restored elsewhere, or another one, is the run's to see (sourceMove)
     const { source: _old, ...kept } = was;
     return { config: { ...kept, apiKey: s.apiKey, folder: s.project, remote: true, served: true }, said };
 }
+/**
+ * THE PROJECT'S PIN, read from the project folder (MASON_PROJECT), in the words of what is running (src/lib/project-version.ts):
+ * the image names the image to run, the package the npm line to run, each to the reader who can act on it. Before
+ * anything is written or connected, so a project of another release line leaves a volume as it found it. A build that
+ * is no release checks nothing.
+ */
+export function servePin(s, running) {
+    const p = (running.from === "image" ? imagePinProblem : commandPinProblem)(s.project, running.text);
+    return p && { message: p.problem, code: "refused", fix: p.ways.join(", or ") || `correct ${PIN_FILE} in the project folder (MASON_PROJECT), then start it again` };
+}
 /**
  * THE PROJECT FOLDER'S SOURCES FILE, for a serve: one there is checked as the run reads it (src/cli/sources.ts
  * readSource: its URL resolved from this environment, every mistake on one report); a folder with none gets the file
@@ -206,36 +260,75 @@ export function serveSource(s, env) {
         wrote = file;
     }
     const r = readSource(s.project, s.data, env, { shellFirst: true });
-    return "message" in r ? r : { wrote };
+    return "message" in r ? r : { wrote, database: connectDatabase(r, { served: true }, env) };
+}
+/**
+ * WHAT THE VOLUME'S CONFIG RECORDS AND THE SOURCES FILE DOES NOT SAY (src/cli/replication-mode.ts recordRefusal), asked
+ * BEFORE `serveSource` and the config are written: a file a serve writes itself says `sync: off`, no one's choice, and a
+ * refused serve leaves the volume and the project folder as it found them. A file that cannot be read, and a project a
+ * serve does not take over, are said by the steps after this.
+ */
+export function serveRecordRefusal(s, existing, env) {
+    if (existing === null || "broken" in existing || existing.config.served !== true)
+        return null;
+    const written = sourceFileNames(s.project).length > 0;
+    if (!written && fs.existsSync(path.join(s.project, OLD_TABLES_DIR)))
+        return null;
+    const file = written ? readSource(s.project, s.data, env, { shellFirst: true }) : null;
+    if (file !== null && "message" in file)
+        return null;
+    return recordRefusal(existing.config, file, env, sourcesTop(path.dirname(s.project), s.project).shown, "mason serve");
 }
+/** seconds since the serve started: /healthz's `uptimeSeconds` and /metrics' `mason_uptime_seconds`, one number */
+const uptimeSeconds = (startedAt, now) => Math.floor((now - startedAt) / 1000);
 /**
  * /healthz's answer: 200 once the run answers, whatever its copies are doing (a table still copying, or one that failed,
  * leaves the rest served: the counts say it); 503 while it starts and stops. Counts only, never a table's name: the
  * endpoint takes no credential.
  */
 export function healthBody(state, startedAt, now = Date.now()) {
-    const base = { status: state.phase, version: version(), uptimeSeconds: Math.floor((now - startedAt) / 1000) };
+    const base = { status: state.phase, version: version(), uptimeSeconds: uptimeSeconds(startedAt, now) };
     if (state.phase !== "ready")
         return { status: 503, body: base };
-    const counts = { total: 0, copied: 0, copying: 0, waiting: 0, failed: 0 };
-    for (const t of state.tables()) {
-        counts.total++;
-        counts[t.phase]++;
-    }
-    return { status: 200, body: { ...base, tables: counts } };
+    return { status: 200, body: { ...base, tables: phaseCounts(state.tables()) } };
 }
-/** The health endpoint: GET /healthz, nothing else. */
-export function healthServer(state, startedAt) {
+/**
+ * /metrics' answer: ALWAYS 200 (a scrape that errors while Mason starts is an alert about nothing; whether it is up is
+ * /healthz's to say) — `mason_ready 0` and `mason_info` while it starts or stops, the run's numbers once it serves.
+ */
+export async function metricsBody(state, startedAt, now = Date.now()) {
+    if (state.phase !== "ready")
+        return metricsText({ version: version(), run: null });
+    const run = await state.metrics();
+    return metricsText({ version: version(), run: { ...run, uptimeSeconds: uptimeSeconds(startedAt, now) } });
+}
+/** The health server: GET /healthz and GET /metrics, nothing else. `log` says a scrape whose numbers could not be read. */
+export function healthServer(state, startedAt, log = say) {
     return http.createServer((req, res) => {
         const url = new URL(req.url ?? "/", "http://health");
-        if (url.pathname !== "/healthz") {
-            res.writeHead(404, { "content-type": "text/plain" }).end("GET /healthz\n");
+        if (url.pathname !== "/healthz" && url.pathname !== "/metrics") {
+            res.writeHead(404, { "content-type": "text/plain" }).end("GET /healthz or GET /metrics\n");
             return;
         }
         if (req.method !== "GET" && req.method !== "HEAD") {
             res.writeHead(405, { allow: "GET, HEAD" }).end();
             return;
         }
+        if (url.pathname === "/metrics") {
+            const asked = state();
+            metricsBody(asked, startedAt).then((body) => { res.writeHead(200, { "content-type": METRICS_CONTENT_TYPE, "cache-control": "no-store" }).end(req.method === "HEAD" ? undefined : body); }, (e) => {
+                // a run stopping under the scrape (its copy closing): said as stopping, as it now is
+                if (state().phase !== "ready") {
+                    res.writeHead(200, { "content-type": METRICS_CONTENT_TYPE, "cache-control": "no-store" }).end(req.method === "HEAD" ? undefined : metricsText({ version: version(), run: null }));
+                    return;
+                }
+                // anything else fails the scrape (Prometheus's `up` turns 0) and is said in the log; the error's text stays in
+                // the log, never in the answer, which takes no credential (it could name a table or a host)
+                log(`  /metrics: the run's numbers could not be read: ${e.message}`);
+                res.writeHead(500, { "content-type": "text/plain", "cache-control": "no-store" }).end("the run's numbers could not be read: mason serve's log says why\n");
+            });
+            return;
+        }
         const h = healthBody(state(), startedAt);
         res.writeHead(h.status, { "content-type": "application/json", "cache-control": "no-store" }).end(`${JSON.stringify(h.body)}\n`);
     });
@@ -251,11 +344,18 @@ export async function serve(argv) {
     const s = serveSettings(process.env, process.cwd());
     if ("message" in s)
         return failed("serve", s, { json });
+    const pinned = servePin(s, runningVersion());
+    if (pinned)
+        return failed("serve", pinned, { json });
+    const existing = readProject(s.data);
+    const unsaid = serveRecordRefusal(s, existing, process.env);
+    if (unsaid !== null)
+        return failed("serve", unsaid, { json });
     // the project folder: its sources file is the list of what is copied, one with no table written when there is none
     const made = serveSource(s, process.env);
     if ("message" in made)
         return failed("serve", made, { json });
-    const c = serveConfig(s, readProject(s.data));
+    const c = serveConfig(s, existing);
     if ("message" in c)
         return failed("serve", c, { json });
     try {
@@ -281,13 +381,15 @@ export async function serve(argv) {
         say(`  wrote ${made.wrote}: the database (url: \${DATABASE_URL}), no table yet (mason add beside it adds one)`);
     for (const l of c.said)
         say(`  ${l}`);
+    say(`  database  ${made.database}, what a tool connects with`);
     say(`  health    http://${s.health.host}:${s.health.port}/healthz`);
+    say(`  metrics   http://${s.health.host}:${s.health.port}/metrics`);
     // the run reads the project of the folder it runs in (src/cli/project.ts): the data folder's
     process.chdir(s.data);
     let code;
     try {
-        code = await dev({ port: 4039, version: version(), changes: s.changes, publication: null, selfSignedTls: s.selfSignedTls,
-            serve: { log: say, ready: (run) => { state = { phase: "ready", tables: run.tables }; }, stopping: () => { state = { phase: "stopping" }; } } });
+        code = await dev({ port: 4039, version: version(), selfSignedTls: s.selfSignedTls,
+            serve: { log: say, ready: (run) => { state = { phase: "ready", tables: run.tables, metrics: run.metrics }; }, stopping: () => { state = { phase: "stopping" }; } } });
     }
     finally {
         // a platform's health check may hold a keep-alive connection open, which close() alone would wait for