@mason-data/cli 0.7.0 → 0.8.0

package/dist/src/cli/source-edit.js

@@ -13,7 +13,8 @@
  */
 import * as fs from "node:fs";
 import * as path from "node:path";
-import { addedTables, chooseColumns, entryName, entryProblems, findEntry, parseSourceText, sourceFailure, sourceFileNames, sourcesDir, unaddableReason, withoutTable, withTable } from "./sources.js";
+import { addedTables, chooseColumns, effectiveSync, entryName, entryProblems, findEntry, parseSourceText, sourceFailure, sourceFileNames, sourcesDir, unaddableReason, withEntrySync, withoutTable, withTable } from "./sources.js";
+import { BIG_TABLE_ROWS, COSTLY_POLL_ROWS, COSTLY_POLL_SECONDS, syncText } from "../lib/table-sync.js";
 /** A refusal the command shows as it is; `failure` is how the web page says it (the CLI's Failure: why, fix, next), by
  *  default the message alone under the status's code. */
 export class Refusal extends Error {
@@ -23,6 +24,31 @@ export class Refusal extends Error {
 }
 /** The FAILURE_CODES code (src/cli/command.ts) of a refusal by its HTTP status. */
 const codeOf = (status) => status === 400 ? "usage" : status === 404 ? "not_found" : status >= 500 ? "failed" : "refused";
+/**
+ * A SCAN'S RATE WHEN NOTHING OF THE TABLE IS MEASURED, rows a second on one backend: the test server's production
+ * replica scanned a 22 M-row table in 12 s at every poll (docs/DESIGN.md "The poll") — an assumption, said as one.
+ */
+const ASSUMED_SCAN_ROWS_PER_SECOND = 1_800_000;
+/**
+ * WHAT A POLL OF A BIG TABLE WOULD COST THE DATABASE, estimated before it runs (issue 33: "a poll never takes more than
+ * a fifth of the source's time" bounds the share, not the backend's seconds — a 200 M-row table set to poll 5m is
+ * still a standing load): seconds a poll, from the best basis there is — its last poll, measured; else its copy's read
+ * rate, a reader's (an upper bound: a poll sends only the changed rows); else the catalog's row estimate at an assumed
+ * scan rate — and the basis said. Null for a table under `COSTLY_POLL_ROWS`, or set to poll every minute or slower.
+ */
+export function pollCost(t, sync, m) {
+    if (sync.mode !== "poll" || sync.seconds >= COSTLY_POLL_SECONDS || t.rows === null || t.rows < COSTLY_POLL_ROWS)
+        return null;
+    const rows = t.rows.toLocaleString("en-US");
+    const [seconds, basis] = m?.lastPollMs != null ? [m.lastPollMs / 1000, "its last poll, measured"]
+        : m?.copyRowsPerSecond ? [t.rows / m.copyRowsPerSecond, `its first sync's read rate, ${Math.round(m.copyRowsPerSecond).toLocaleString("en-US")} rows a second a reader (a poll sends only the rows that changed, so it is at most that)`]
+            : [t.rows / ASSUMED_SCAN_ROWS_PER_SECOND, `the catalog's ${rows} rows at an assumed ${ASSUMED_SCAN_ROWS_PER_SECOND.toLocaleString("en-US")} rows a second of scan (nothing of it measured yet)`];
+    const every = Math.max(sync.seconds, 4 * seconds);
+    const s = (x) => x < 90 ? `${Math.round(x)} s` : x < 5400 ? `${Math.round(x / 60)} min` : `${(x / 3600).toFixed(1)} h`;
+    return `${entryName(t.source)} has about ${rows} rows, and each poll scans it whole on your database: about ${s(seconds)} a poll, from ${basis}. ` +
+        `sync: ${syncText(sync)} is stretched to a poll every ${s(every)} at most (a poll waits four times its own time), one backend of your database ` +
+        `busy ${Math.round((seconds / every) * 100)} % of the time — a longer interval (--sync "poll 1h"), or --sync replication (the change log), costs it less`;
+}
 /**
  * The project's sources file as it is on disk NOW, its text and what it says — or why it cannot be edited: no folder, not
  * exactly one file, a file with mistakes of its own, a URL that cannot be resolved (`read`, the file read as the run reads
@@ -49,7 +75,7 @@ export function sourceToEdit(folder, root, read) {
 /** The file's other entries against the database: an edit is never written over a file the run would not take. */
 function othersHold(at, db, without, then) {
     const others = { ...at.parsed, tables: at.parsed.tables.filter((e) => !without.includes(e.table)) };
-    const held = addedTables(others, db.tables, db.database, at.root, { replication: db.replication });
+    const held = addedTables(others, db.tables, db.database, at.root);
     if (!Array.isArray(held))
         throw new Refusal(409, `${held.message}${held.why ? `\n${held.why}` : ""} — fix it first, then ${then}`);
 }
@@ -58,35 +84,61 @@ function othersHold(at, db, without, then) {
  * table's own (`withTable`): every other byte of the file as it was. `columns` are the ones asked (`["*"]`: every one);
  * `key` the columns a table with no primary key is copied by. The entry and the file's other entries must hold against
  * the database first: an edit is never written over a file the next read would refuse.
+ *
+ * ITS SYNC (`o.sync`, --sync): written on the entry as its own `sync:`; else an entry's own is kept; else a NEW entry
+ * follows the file's default — unless the table is over `BIG_TABLE_ROWS` and that default polls, when the entry is
+ * written `sync: off` and said (`bigOff`): a poll scans its table whole — or the default is `replication` and the table
+ * cannot be (a `--key` table, REPLICA IDENTITY NOTHING), when it is written `sync: poll 5m` and said why (`notReplicated`;
+ * an explicit `--sync replication` of such a table is refused, at its entry). `o.keep`: only the sync is asked of a table
+ * already added (`mason add orders --sync "poll 30s"`), so its columns and key stay as they are, and only its `sync:`
+ * line is written (`withEntrySync`: its comments too).
  */
-export function addEntry(at, db, table, columns = ["*"], key = null) {
+export function addEntry(at, db, table, columns = ["*"], key = null, o = {}) {
     const t = db.tables.find((x) => x.source === table);
     if (!t)
         throw new Refusal(404, `${table} is not a table of ${db.database} that this user can read`);
     if (key !== null && t.key.length)
-        throw new Refusal(400, `${table} has a primary key (${t.key.join(", ")}), which Mason copies it by: leave --key out`);
+        throw new Refusal(400, `${table} has a primary key (${t.key.join(", ")}), which Mason keys it by: leave --key out`);
     if (key === null && !t.key.length)
         throw new Refusal(409, `${table} cannot be added: ${unaddableReason(t)}`);
-    if (key !== null && db.replication)
-        throw new Refusal(409, `${table} has no primary key, and logical replication cannot apply its changes by --key: copy it by polling (mason start --poll), then add it`);
-    const chosen = chooseColumns(t, columns, key);
+    const was = at.parsed.tables.find((e) => e.table === table);
+    const keep = o.keep === true && was !== undefined;
+    const chosen = keep ? { columns: was.columns, exclude: was.exclude, addedKey: [], skipped: [] } : chooseColumns(t, columns, key);
     if ("problem" in chosen)
         throw new Refusal(400, chosen.problem);
     othersHold(at, db, [table], `add ${entryName(table)}`);
-    const was = at.parsed.tables.find((e) => e.table === table);
-    const entry = { name: was?.name ?? entryName(table), table, columns: chosen.columns, exclude: chosen.exclude, key: key === null ? null : [...key] };
+    // its own sync: asked, else the one its entry has, else none (the file's) — but a new big table under a polling default
+    const fileSync = effectiveSync(at.parsed, undefined).sync;
+    const bigOff = o.sync === undefined && was === undefined && fileSync.mode === "poll" && t.rows !== null && t.rows > BIG_TABLE_ROWS;
+    // a table logical replication cannot keep, under a file whose default is replication: polled, said (never refused for
+    // the default it did not choose; its own --sync replication is refused at its entry, below)
+    const keyed = keep ? was.key !== null : key !== null;
+    const unreplicable = keyed ? "it has no primary key, so logical replication cannot apply its changes by its key:"
+        : t.replicaIdentity === "n" ? "it has REPLICA IDENTITY NOTHING, and a publication carrying it would make your app's own UPDATEs and DELETEs on it fail" : null;
+    const notReplicated = o.sync === undefined && (was?.sync ?? null) === null && fileSync.mode === "replication" ? unreplicable : null;
+    // --sync replication asked of one: refused with the COMMAND to run (the person is at it), its own flags kept — the
+    // file's words are for an entry written by hand (src/cli/sources.ts entryProblems)
+    if (o.sync?.mode === "replication" && unreplicable !== null) {
+        const others = columns.filter((c) => c !== "*");
+        const flags = keep ? "" : `${columns.includes("*") ? (others.length ? ` --include ${others.join(",")}` : "") : ` --columns ${others.join(",")}`}${key !== null ? ` --key ${key.join(",")}` : ""}`;
+        throw new Refusal(400, `${table} cannot be kept current by logical replication: ${unreplicable} — mason add ${entryName(table)}${flags} --sync "poll 5m" polls it instead`);
+    }
+    const sync = o.sync ?? was?.sync ?? (bigOff ? { mode: "off" } : notReplicated !== null ? { mode: "poll", every: "5m", seconds: 300 } : null);
+    const entry = { name: was?.name ?? entryName(table), table, columns: chosen.columns, exclude: chosen.exclude, key: keep ? was.key : key === null ? null : [...key], sync };
     // the entry itself, as the run will read it: a --key that does not tell the rows apart is refused here, not at the copy
-    const own = entryProblems(t, entry, db.tables, db.database, { replication: db.replication });
+    const own = entryProblems(t, entry, db.tables, db.database, effectiveSync(at.parsed, entry).sync);
     if (own.length)
         throw new Refusal(400, own.join("; "));
-    const same = was !== undefined && JSON.stringify([was.columns, was.exclude, was.key]) === JSON.stringify([entry.columns, entry.exclude, entry.key]);
+    const same = was !== undefined && JSON.stringify([was.columns, was.exclude, was.key, was.sync]) === JSON.stringify([entry.columns, entry.exclude, entry.key, entry.sync]);
     if (!same) {
-        const r = withTable(at.text, entry);
+        const r = keep ? withEntrySync(at.text, was, sync) : withTable(at.text, entry);
         if ("problem" in r)
             throw new Refusal(409, `${path.relative(at.root, at.file)} cannot be edited: ${r.problem}`);
         fs.writeFileSync(at.file, r.text);
     }
-    return { table, file: at.file, entry: entry.name, columns: entry.columns, exclude: entry.exclude, key: entry.key, addedKey: chosen.addedKey, skippedSecrets: chosen.skipped, already: same };
+    const eff = effectiveSync(at.parsed, entry);
+    return { table, file: at.file, entry: entry.name, columns: entry.columns, exclude: entry.exclude, key: entry.key, addedKey: chosen.addedKey, skippedSecrets: chosen.skipped, already: same,
+        sync: { value: syncText(eff.sync), from: eff.from }, bigOff, notReplicated, pollCost: pollCost(t, eff.sync, db.measured?.(table) ?? null) };
 }
 /**
  * REMOVE TABLES from the sources file: each one's entry cut (`withoutTable`), every other byte of the file as it was. A

package/dist/src/cli/sources.js

@@ -5,9 +5,16 @@
  *
  *   kind: postgres
  *   url: ${DATABASE_URL}          an env var from .env.local, .env or the shell; a literal URL for local dev
+ *   database: production          the database name a tool connects with (none: mason; src/lib/database-name.ts)
+ *   sync: off                     how each table is kept current after its first sync (src/lib/table-sync.ts): off
+ *                                 (the default: until mason sync <table>), poll <interval> or replication; an entry's
+ *                                 own overrides it
+ *   publication: ${PG_PUBLICATION}  the publication `replication` reads, its owner's (never altered); none: Mason's own
+ *                                   (the literal `publication: null` says so, past one an earlier start recorded)
  *   tables:
  *     orders:                     a bare name is in `public`; `billing.invoices` names its schema
  *       columns: [id, customer_id, status]
+ *       sync: poll 30s            this table polled every 30 seconds
  *     customers: {}               every column
  *     events:
  *       exclude: [raw_body]       every column but these
@@ -26,13 +33,40 @@
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { isMap, isScalar, parseDocument } from "yaml";
+import { databaseNameProblem, DEFAULT_DATABASE } from "../lib/database-name.js";
 import { appEnvValue, ENV_FILE } from "../lib/dotenv.js";
 import { closest, didYouMean } from "../lib/did-you-mean.js";
 import { maskedUrl } from "./preflight.js";
+import { DEFAULT_SYNC, parseSync, syncText } from "../lib/table-sync.js";
 export const SOURCES_DIR = "sources";
-const KEYS = ["kind", "url", "tables"];
-const ENTRY_KEYS = ["columns", "exclude", "key"];
+const KEYS = ["kind", "url", "database", "sync", "publication", "tables"];
+const ENTRY_KEYS = ["columns", "exclude", "key", "sync"];
 const ENV_REF = /^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$/;
+/** The tables a file keeps current by replication, by their effective sync (`effectiveSync`): what streams. */
+export function replicatedEntries(src) {
+    return src.tables.filter((e) => effectiveSync(src, e).sync.mode === "replication");
+}
+/** Whether the file says how anything is kept current: a `sync:` at its top or on any entry. A file with none has every
+ *  table on the default, by no one's choice: what an older Mason's file looks like. */
+export function syncWritten(file) {
+    return file.sync !== null || file.tables.some((e) => e.sync !== null);
+}
+/**
+ * A FILE THAT HAS GONE SILENT OVER A PROJECT THAT STREAMS: it names tables and no `sync:`, where the config records a
+ * slot. Silent is no decision (a file with no table has nothing left to stream). One rule for the start, whose refusal
+ * is `recordRefusal` (src/cli/replication-mode.ts), and the run's re-read (`rereadSources`).
+ */
+export function silentOverSlot(slotRecorded, file) {
+    return slotRecorded && file.tables.length > 0 && !syncWritten(file);
+}
+/** A table's sync: its entry's, else the file's, else the default (`off`) — and which of the three said it. */
+export function effectiveSync(src, e) {
+    if (e?.sync)
+        return { sync: e.sync, from: "entry" };
+    if (src.sync)
+        return { sync: src.sync, from: "file" };
+    return { sync: DEFAULT_SYNC, from: "default" };
+}
 /** A file as messages name it: relative to the project's root. */
 export const shownPath = (root, file) => path.relative(root, file) || file;
 /** A table as a key of the file: its name alone in public, else schema.table. */
@@ -90,6 +124,25 @@ function nameList(v, at, what) {
         return `${at}: names ${String(twice)} twice`;
     return v;
 }
+/** The file's `database:`: the name it writes, null when it has no such line, or the problem with it (`text`, the file's). */
+function databaseKey(doc, text) {
+    const node = isMap(doc.contents) ? doc.contents.get("database", true) : undefined;
+    if (node === undefined)
+        return { database: null };
+    const way = `write database: <name>, or take the line out: tools then connect with ${DEFAULT_DATABASE}`;
+    if (!isScalar(node))
+        return { problem: `database: is one name, not a list or a map — ${way}` };
+    if (node.value === null)
+        return { problem: `database: is empty — ${way}` };
+    if (typeof node.value !== "string") {
+        const written = text.slice(node.range[0], node.range[1]);
+        return { problem: `database: ${yamlRead(node.value).replace(/^\S+/, written)} — quote it: "${written}"` };
+    }
+    const bad = databaseNameProblem(node.value);
+    return bad === null ? { database: node.value } : { problem: `database: ${bad} — ${way}` };
+}
+/** A file that is YAML and no mapping, said; `v` is what it read as. */
+const notMapping = (v) => `it is ${v === null || v === undefined ? "empty" : "not a mapping"}: a source file is kind: postgres, url: \${DATABASE_URL} and tables: {}`;
 /**
  * The file's text read as a source file: its shape, never the database (that is `addedTables`'s). `name` is the
  * source's (the file's name); the problems are each at their key, in the order they are in the file.
@@ -100,7 +153,7 @@ export function parseSourceText(text, name, file) {
         return { problems: [`it is not YAML: ${doc.errors[0].message.split("\n")[0]}`] };
     const v = doc.toJS();
     if (!isPlain(v))
-        return { problems: [`it is ${v === null || v === undefined ? "empty" : "not a mapping"}: a source file is kind: postgres, url: \${DATABASE_URL} and tables: {}`] };
+        return { problems: [notMapping(v)] };
     const problems = [];
     for (const k of Object.keys(v)) {
         if (!KEYS.includes(k))
@@ -132,6 +185,38 @@ export function parseSourceText(text, name, file) {
         else
             url = { literal: v.url.trim() };
     }
+    // the name a tool connects with: written out, the same on every machine (src/lib/database-name.ts)
+    const named = databaseKey(doc, text);
+    if ("problem" in named)
+        problems.push(named.problem);
+    // the publication a replicated table is read through: a name, or ${NAME} as url: takes one (resolved by readSource)
+    let publication = null;
+    // the word `null`, as written: Mason's own, on purpose, past a name a start recorded (src/cli/replication-mode.ts
+    // recordRefusal). A key left with no value is a line half-written, never that decision; no line at all says nothing
+    const publicationNode = doc.get("publication", true);
+    if (v.publication === null && isScalar(publicationNode) && publicationNode.source === "null")
+        publication = { own: true };
+    else if (v.publication !== undefined) {
+        const p = typeof v.publication === "string" ? v.publication.trim() : "";
+        const m = ENV_REF.exec(p);
+        if (p === "")
+            problems.push("publication: write publication: ${PG_PUBLICATION}, or a publication's name (publication: null and Mason makes its own)");
+        else if (m)
+            publication = { env: m[1] };
+        else if (p.includes("$"))
+            problems.push(`publication: ${JSON.stringify(p)} — write \${NAME} alone, like \${PG_PUBLICATION}, or the publication's name`);
+        else
+            publication = { literal: p };
+    }
+    // every table's sync, unless its entry says its own (src/lib/table-sync.ts: one grammar, `mason add --sync` reads it too)
+    let sync = null;
+    if (v.sync !== undefined) {
+        const s = parseSync(v.sync);
+        if ("problem" in s)
+            problems.push(`sync: ${s.problem}`);
+        else
+            sync = s;
+    }
     const tables = [];
     const tablesNode = isMap(doc.contents) ? doc.contents.items.find((p) => isScalar(p.key) && p.key.value === "tables")?.value : undefined;
     const odd = new Set();
@@ -167,30 +252,33 @@ export function parseSourceText(text, name, file) {
             }
             seen.set(table, key);
             if (raw !== null && !isPlain(raw)) {
-                problems.push(`${at}: is {} (every column), { columns: [...] }, { exclude: [...] } or { key: [...] }`);
+                problems.push(`${at}: is {} (every column), { columns: [...] }, { exclude: [...] }, { key: [...] } or { sync: poll 5m }`);
                 continue;
             }
             const e = raw ?? {};
             const stray = Object.keys(e).filter((k) => !ENTRY_KEYS.includes(k));
             for (const k of stray)
-                problems.push(`${at}: unexpected key ${k}${didYouMean(k, ENTRY_KEYS)} — a table takes columns, exclude and key`);
+                problems.push(`${at}: unexpected key ${k}${didYouMean(k, ENTRY_KEYS)} — a table takes columns, exclude, key and sync`);
             const both = e.columns !== undefined && e.exclude !== undefined;
             if (both)
-                problems.push(`${at}: columns and exclude together — columns names what is copied, exclude what is left out of every column; keep one`);
-            const columns = e.columns === undefined ? null : nameList(e.columns, `${at}.columns`, "the columns copied");
+                problems.push(`${at}: columns and exclude together — columns names what is synced, exclude what is left out of every column; keep one`);
+            const columns = e.columns === undefined ? null : nameList(e.columns, `${at}.columns`, "the columns synced");
             const exclude = e.exclude === undefined ? [] : nameList(e.exclude, `${at}.exclude`, "the columns left out");
             const ownKey = e.key === undefined ? null : nameList(e.key, `${at}.key`, "the columns that tell its rows apart");
             for (const x of [columns, exclude, ownKey])
                 if (typeof x === "string")
                     problems.push(x);
-            if (typeof columns === "string" || typeof exclude === "string" || typeof ownKey === "string" || stray.length || both)
+            const ownSync = e.sync === undefined ? null : parseSync(e.sync);
+            if (ownSync !== null && "problem" in ownSync)
+                problems.push(`${at}.sync: ${ownSync.problem}`);
+            if (typeof columns === "string" || typeof exclude === "string" || typeof ownKey === "string" || stray.length || both || (ownSync !== null && "problem" in ownSync))
                 continue;
-            tables.push({ name: key, table, columns, exclude, key: ownKey });
+            tables.push({ name: key, table, columns, exclude, key: ownKey, sync: ownSync });
         }
     }
-    if (problems.length || kind === null || url === null)
+    if (problems.length || kind === null || url === null || "problem" in named)
         return { problems };
-    return { name, file, kind, url, tables };
+    return { name, file, kind, url, database: named.database, sync, publication, tables };
 }
 /** Where a project's source files are: `<folder>/sources/`. */
 export const sourcesDir = (folder) => path.join(folder, SOURCES_DIR);
@@ -201,16 +289,80 @@ export function sourceFileNames(folder) {
         return [];
     return fs.readdirSync(dir).filter((f) => /\.ya?ml$/.test(f) && !f.startsWith(".")).sort();
 }
-/** The file `mason init` writes: the database, then its tables — none yet, but for `mason init demo`'s, which names the
- *  demo database's tables (`tables`, each copied whole: `{}`), so its first start copies them and shows something. */
-export function newSourceText(url = "${DATABASE_URL}", tables = []) {
-    return "# The database this project copies from, and which of its tables. Yours to commit; the URL stays out of it.\n" +
+/**
+ * THE TOP OF A PROJECT'S SOURCES FILE, for a retired setting's way out (src/lib/table-sync.ts `topEdit`): its one file
+ * as a person names it from `root`, and the values its top-level `sync:` and `publication:` lines hold as written, so
+ * the way out changes the line the file has rather than writing a second. No file (or several): the place every
+ * project's file has, and no line. A file that cannot be read or parsed is said by the run in its own words
+ * (`readSource`); here it only words the edit, as for a file with no such line.
+ */
+export function sourcesTop(root, folder) {
+    const names = sourceFileNames(folder);
+    if (names.length !== 1)
+        return { shown: `${shownPath(root, sourcesDir(folder))}/<name>.yml`, sync: null, publication: null };
+    const file = path.join(sourcesDir(folder), names[0]);
+    const shown = shownPath(root, file);
+    let text;
+    try {
+        text = fs.readFileSync(file, "utf8");
+    }
+    catch (e) {
+        // the run's own read says it, with the file and the error (readSource); the edit is worded as for no line
+        if (e.code === undefined)
+            throw e;
+        return { shown, sync: null, publication: null };
+    }
+    const doc = parseDocument(text);
+    const at = (key) => {
+        if (!isMap(doc.contents))
+            return null;
+        const node = doc.contents.get(key, true);
+        if (node === undefined || node === null)
+            return null;
+        return isScalar(node) ? String(node.value) : String(node);
+    };
+    return { shown, sync: at("sync"), publication: at("publication") };
+}
+/**
+ * THE `database:` A PROJECT WRITES, from the file alone and only its one key (src/lib/database-name.ts): what `mason user
+ * add` says with Mason stopped, which needs neither the database's URL nor the rest of the file right. No file writes
+ * none. A file it cannot read this key from is refused, in the failure shape: guessing `mason` would hand a tool a name
+ * the run may not answer to.
+ */
+export function writtenDatabase(folder, root) {
+    const names = sourceFileNames(folder);
+    if (names.length === 0)
+        return { database: null };
+    if (names.length > 1)
+        return tooManySources(root, sourcesDir(folder), names);
+    const file = path.join(sourcesDir(folder), names[0]);
+    const text = fs.readFileSync(file, "utf8");
+    const doc = parseDocument(text, { uniqueKeys: true });
+    if (doc.errors.length)
+        return sourceFailure(file, root, [`it is not YAML: ${doc.errors[0].message.split("\n")[0]}`]);
+    if (!isMap(doc.contents))
+        return sourceFailure(file, root, [notMapping(doc.toJS())]);
+    const named = databaseKey(doc, text);
+    return "problem" in named ? sourceFailure(file, root, [named.problem]) : named;
+}
+/** The file `mason init` writes: the database, how its tables are kept current (`sync`: `off`, the default; the demo's
+ *  `poll 5s`, so it shows a copy keeping up), then its tables — none yet, but for `mason init demo`'s, which names the
+ *  demo database's tables (`tables`, each copied whole: `{}`), so its first start copies them and shows something.
+ *  `database:` only when one is asked for (`--database`): none written, the default stays a default. */
+export function newSourceText(url = "${DATABASE_URL}", tables = [], sync = DEFAULT_SYNC, database = null) {
+    return "# The database this project syncs from, and which of its tables. Yours to commit; the URL stays out of it.\n" +
+        (database === null ? `# Tools connect with the database name ${DEFAULT_DATABASE}; database: <name> here keeps the name a tool already has.\n` : "") +
         "kind: postgres\n" +
         `url: ${url}${ENV_REF.test(url) ? "   # from .env.local, .env or the shell; a literal postgres:// URL works too" : ""}\n` +
+        (database === null ? "" : `database: ${yamlName(database)}   # the database name a tool connects with\n`) +
+        `sync: ${syncText(sync)}${sync.mode === "off" ? "   # after its first sync, a table is read again only by mason sync <table> (poll 5m: every 5 minutes)"
+            : sync.mode === "replication" ? "   # each table kept current from the database's change log after its first sync (poll 5m, or off, on an entry)"
+                : `   # each table polled every ${sync.every} after its first sync (off: read again only by mason sync <table>)`}\n` +
         "\n" +
-        "# One entry per table, a bare name in public, else schema.table: {} copies every column, columns: [id, status]\n" +
+        "# One entry per table, a bare name in public, else schema.table: {} syncs every column, columns: [id, status]\n" +
         "# those, exclude: [payload] every column but those; key: [a, b] names the columns that tell its rows apart, for a\n" +
-        "# table with no primary key. mason add and mason remove edit this list; so can you.\n" +
+        "# table with no primary key; sync: poll 30s (or off) keeps that table current its own way. mason add and mason remove\n" +
+        "# edit this list; so can you.\n" +
         (tables.length ? `tables:\n${tables.map((t) => `  ${t}: {}\n`).join("")}` : "tables: {}\n");
 }
 /** A source's name from its database's: the file is named for it, else `app`. */
@@ -220,10 +372,15 @@ export function sourceNameFor(database) {
 }
 /** The folder an older Mason listed a project's tables in, one file per table: read by nothing, said by `readSource`. */
 export const OLD_TABLES_DIR = "models";
+/** A second source file, refused: one source per project for now. */
+const tooManySources = (root, dir, names) => ({
+    message: `${shownPath(root, dir)}/ has ${names.length} source files (${names.join(", ")}): one source per project for now`, code: "bad_source",
+    fix: `keep the one this project syncs from, and move the others out of ${shownPath(root, dir)}/`
+});
 /**
  * THE PROJECT'S SOURCE FILE, read: the one file in `sources/` (a second is refused by name: one source per project for
  * now), parsed, its URL resolved — `${NAME}` from the project's .env.local or .env, else the shell (src/lib/dotenv.ts
- * `appEnvValue`, the order `mason init` reads DATABASE_URL in, so the two cannot disagree); `shellFirst` for a server
+ * `appEnvValue`, the order `mason init` reads MASON_DATABASE_URL and DATABASE_URL in, so the two cannot disagree); `shellFirst` for a server
  * (`mason serve`), configured by its environment alone. A project with no file is told what to write — one an older
  * Mason made (its tables listed in another folder, which nothing reads) that mason init writes it.
  */
@@ -241,23 +398,35 @@ export function readSource(folder, root, env = process.env, o = {}) {
         return { message: `this project names no database: ${shownPath(root, dir)}/ has no source file`, code: "bad_source",
             fix: `write ${shownPath(root, file)}, with the database's URL in .env as DATABASE_URL:\n${newSourceText().trimEnd()}` };
     }
-    if (names.length > 1) {
-        return { message: `${shownPath(root, dir)}/ has ${names.length} source files (${names.join(", ")}): one source per project for now`, code: "bad_source",
-            fix: `keep the one this project copies from, and move the others out of ${shownPath(root, dir)}/` };
-    }
+    if (names.length > 1)
+        return tooManySources(root, dir, names);
     const file = path.join(dir, names[0]);
     const parsed = parseSourceText(fs.readFileSync(file, "utf8"), names[0].replace(/\.ya?ml$/, ""), file);
     if ("problems" in parsed)
         return sourceFailure(file, root, parsed.problems);
+    const where = o.shellFirst ? "the environment, .env.local or .env" : ".env.local, .env or the shell";
+    // the publication's name, read as url: is: ${NAME} from the same places, in the same order
+    let publicationName = null;
+    if (parsed.publication !== null && "literal" in parsed.publication)
+        publicationName = parsed.publication.literal;
+    else if (parsed.publication !== null && "env" in parsed.publication) {
+        const v = parsed.publication.env;
+        const p = appEnvValue(root, v, env, { shellFirst: o.shellFirst === true });
+        if (p === null || p.value.trim() === "") {
+            return { message: `${shownPath(root, file)}: publication: \${${v}} is not set — not in ${where}`, code: "bad_source",
+                fix: `put ${v}=<the publication's name> in ${shownPath(root, path.join(root, ENV_FILE))} (or export it), or write publication: null and Mason makes its own` };
+        }
+        publicationName = p.value.trim();
+    }
     if ("literal" in parsed.url)
-        return { ...parsed, resolved: parsed.url.literal, from: shownPath(root, file) };
+        return { ...parsed, resolved: parsed.url.literal, from: shownPath(root, file), publicationName };
     const found = appEnvValue(root, parsed.url.env, env, { shellFirst: o.shellFirst === true });
     if (found === null) {
         const v = parsed.url.env;
-        return { message: `${shownPath(root, file)}: url: \${${v}} is not set — not in ${o.shellFirst ? "the environment, .env.local or .env" : ".env.local, .env or the shell"}`, code: "bad_source",
+        return { message: `${shownPath(root, file)}: url: \${${v}} is not set — not in ${where}`, code: "bad_source",
             fix: `put ${v}=postgres://user:password@host:5432/your_app in ${shownPath(root, path.join(root, ENV_FILE))} (or export it), or write the URL in url:` };
     }
-    return { ...parsed, resolved: found.value, from: `${parsed.url.env} in ${found.from}` };
+    return { ...parsed, resolved: found.value, from: `${parsed.url.env} in ${found.from}`, publicationName };
 }
 /** The columns a table's copy keeps, in the source's order: the entry's, its key always among them. */
 export function copiedColumns(t, e) {
@@ -267,7 +436,7 @@ export function copiedColumns(t, e) {
     return t.columns.map((c) => c.name).filter((c) => keep.has(c));
 }
 /** Why a table cannot be copied as its entry says, each at its key, against the database's catalog; none when it can. */
-export function entryProblems(t, e, all, database, o) {
+export function entryProblems(t, e, all, database, sync) {
     const at = `tables.${e.name}`;
     if (t === undefined) {
         const names = all.map((x) => entryName(x.source));
@@ -278,13 +447,13 @@ export function entryProblems(t, e, all, database, o) {
     const out = [...missing(e.columns ?? [], "columns"), ...missing(e.exclude, "exclude")];
     if (e.key !== null) {
         if (t.key.length)
-            out.push(`${at}.key: ${t.source} has a primary key (${t.key.join(", ")}), which Mason copies it by — remove key`);
+            out.push(`${at}.key: ${t.source} has a primary key (${t.key.join(", ")}), which Mason keys it by — remove key`);
         else {
             out.push(...missing(e.key, "key"));
             out.push(...keyProblems(t, e.key, at));
             // its rows have no identity on the database, so logical replication carries no key to apply a change by
-            if (o.replication)
-                out.push(`${at}: ${t.source} has no primary key, and logical replication cannot apply its changes by key: — copy it by polling (mason start --poll), or remove it`);
+            if (sync.mode === "replication")
+                out.push(`${at}: ${t.source} has no primary key, so logical replication cannot apply its changes by its key: (sync: replication) — give its entry sync: poll <interval> (poll 5m), or sync: off`);
         }
     }
     else if (!t.key.length) {
@@ -293,7 +462,13 @@ export function entryProblems(t, e, all, database, o) {
             `${unique ? ` (its unique index ${unique.name})` : ""}`);
     }
     for (const k of (e.key ?? t.key).filter((k) => e.exclude.includes(k)))
-        out.push(`${at}.exclude: ${k} is ${t.source}'s key, which is always copied — take it out of exclude`);
+        out.push(`${at}.exclude: ${k} is ${t.source}'s key, which is always synced — take it out of exclude`);
+    // a table that identifies no changed row: a publication carrying its updates would make the app's own UPDATEs and
+    // DELETEs on it fail (the catalog says so; the file cannot), so it is never replicated
+    if (sync.mode === "replication" && t.replicaIdentity === "n" && e.key === null) {
+        out.push(`${at}: ${t.source} has REPLICA IDENTITY NOTHING, and a publication carrying it would make your app's own UPDATEs and DELETEs on it fail (sync: replication) — ` +
+            `give its entry sync: poll <interval> (poll 5m), or sync: off; or ALTER TABLE ${t.source} REPLICA IDENTITY DEFAULT on the database`);
+    }
     return out;
 }
 /**
@@ -315,7 +490,7 @@ function keyProblems(t, key, at) {
     if (!(t.uniqueKeys ?? []).some((u) => u.columns.every((c) => key.includes(c)))) {
         out.push(`${at}.key: no unique index of ${t.source} is over (${key.join(", ")}), so two of its rows may share them and Mason could not tell them apart — ` +
             (usable.length ? `name the columns of one: ${usable.map((u) => `key: [${u.columns.join(", ")}] (its unique index ${u.name})`).join(", or ")}`
-                : `it has no unique index over NOT NULL columns, so Mason cannot copy it until its owner adds one (or a primary key)`));
+                : `it has no unique index over NOT NULL columns, so Mason cannot sync it until its owner adds one (or a primary key)`));
     }
     return out;
 }
@@ -325,19 +500,20 @@ function keyProblems(t, key, at) {
  * `primaryKey` keeps the source's own (none): the catalog declares only what the source does. Every entry's problems on
  * one report (`sourceFailure`).
  */
-export function addedTables(src, tables, database, root, o) {
+export function addedTables(src, tables, database, root) {
     const problems = [];
     const out = [];
     for (const e of src.tables) {
         const t = tables.find((x) => x.source === e.table);
-        const p = entryProblems(t, e, tables, database, o);
+        const sync = effectiveSync(src, e).sync;
+        const p = entryProblems(t, e, tables, database, sync);
         if (p.length || t === undefined) {
             problems.push(...p);
             continue;
         }
         const keep = new Set(copiedColumns(t, e));
         out.push({ ...t, columns: t.columns.filter((c) => keep.has(c.name)), foreignKeys: t.foreignKeys.filter((f) => keep.has(f.column)),
-            ...(e.key !== null ? { key: e.key, primaryKey: t.key } : {}) });
+            ...(e.key !== null ? { key: e.key, primaryKey: t.key } : {}), sync });
     }
     return problems.length ? sourceFailure(src.file, root, problems, [["mason tables", "the database's tables and their columns"]]) : out;
 }
@@ -345,18 +521,30 @@ export function addedTables(src, tables, database, root, o) {
  * THE SOURCES FILE READ AGAIN WHILE THE RUN IS UP (mason add, mason remove): its tables now — or, when the file cannot be
  * taken as it is, the tables read before, kept (a typo never drops a copy), with what is wrong; `said` is the log line,
  * once per new problem. A `url:` that resolves to another database than the one the run copies (a .env edited) is such a
- * problem: the run's catalog is the old database's, so it is said, and mason stop && mason start reads the new one.
+ * problem: the run's catalog is the old database's, so it is said, and mason stop && mason start reads the new one. So
+ * is a file gone silent on sync: while `slot` is recorded: taken, it would retire the slot.
  */
-export function rereadSources(prev, read, runUrl, catalog, database, root, o) {
+export function rereadSources(prev, read, runUrl, catalog, database, root, o = {}) {
     let problem = null;
     let next = null;
     if ("message" in read)
         problem = `${read.message}${read.why ? `: ${read.why.replace(/\n/g, "; ")}` : ""}`;
     else if (read.resolved !== runUrl) {
-        problem = `${shownPath(root, read.file)}: url: now names ${maskedUrl(read.resolved)} (${read.from}), and this run copies ${maskedUrl(runUrl)}: mason stop && mason start copies from the new one`;
+        problem = `${shownPath(root, read.file)}: url: now names ${maskedUrl(read.resolved)} (${read.from}), and this run syncs ${maskedUrl(runUrl)}: mason stop && mason start syncs from the new one`;
+    }
+    else if (silentOverSlot((o.slot ?? null) !== null, read)) {
+        problem = `${shownPath(root, read.file)}: no sync: at its top or on an entry, and this project streams through replication slot ${o.slot}: ` +
+            "write sync: replication at its top to keep streaming, or sync: off or sync: poll <interval> to drop the slot";
+    }
+    else if (o.replicationChecked === false && replicatedEntries(read).length > 0) {
+        // a table set to replication in a run that never checked the database for it (the commands' edits are checked, the
+        // run server's checkReplication): kept as it was, never frozen by a slot the database cannot make
+        const names = replicatedEntries(read).map((e) => entryName(e.table));
+        problem = `${shownPath(root, read.file)}: ${names.join(", ")} ${names.length === 1 ? "is" : "are"} sync: replication, and this run has not checked ${database} for logical replication ` +
+            "(no table was replicated when it started): mason stop && mason start checks it and streams them";
     }
     else {
-        const t = addedTables(read, catalog, database, root, o);
+        const t = addedTables(read, catalog, database, root);
         if (Array.isArray(t))
             next = t;
         else
@@ -414,6 +602,8 @@ export function entryText(e, indent) {
         body.push(`exclude: ${flowList(e.exclude)}`);
     if (e.key !== null)
         body.push(`key: ${flowList(e.key)}`);
+    if (e.sync)
+        body.push(`sync: ${syncText(e.sync)}`);
     if (!body.length)
         return `${indent}${yamlName(e.name)}: {}\n`;
     return `${indent}${yamlName(e.name)}:\n${body.map((b) => `${indent}  ${b}\n`).join("")}`;
@@ -498,6 +688,65 @@ function withTableLf(text, e) {
     const at = lineEnd(lv?.range ? lv.range[1] : last.key.range[1]);
     return { text: `${text.slice(0, at)}${at > 0 && text[at - 1] !== "\n" ? "\n" : ""}${entryText(e, indent)}${text.slice(at)}` };
 }
+/**
+ * The file with ONE entry's own `sync:` set (`sync`), or taken out (null: it follows the file's again), and every other
+ * byte as it was — the entry's columns, key and comments too (`mason add orders --sync "poll 30s"` on a table already
+ * added): the value spliced at its node's range, a line of its own under the entry's last key at its keys' indent, or —
+ * an entry written as a flow map (`{}`, `{ columns: [...] }`) — the entry written again a key a line, a comment after it
+ * on the key's line kept. `e` is the entry as the file reads (its columns and key, for a flow map written again).
+ */
+export function withEntrySync(text, e, sync) {
+    return inOwnLineEndings(text, (t) => withEntrySyncLf(t, e, sync));
+}
+function withEntrySyncLf(text, e, sync) {
+    const r = forEdit(text);
+    if ("problem" in r)
+        return r;
+    const { pair, lineStart, lineEnd } = r;
+    const map = pair?.value;
+    if (!isMap(map))
+        return withTableLf(text, { ...e, sync });
+    if (map.flow)
+        return { problem: FLOW_MAP };
+    const same = entryPairs(map).find((x) => x.table === e.table);
+    if (same === undefined)
+        return withTableLf(text, { ...e, sync });
+    const value = same.pair.value;
+    if (isMap(value) && !value.flow && value.items.length > 0) {
+        const own = value.items.find((p) => isScalar(p.key) && p.key.value === "sync");
+        if (own !== undefined) {
+            const k = own.key, v = own.value;
+            // its line cut, unless it is the entry's only key: then the entry is `{}` again
+            if (sync === null && value.items.length === 1)
+                return withTableLf(text, { ...e, name: same.name, sync: null });
+            if (sync === null)
+                return { text: `${text.slice(0, lineStart(k.range[0]))}${text.slice(lineEnd(v?.range ? v.range[1] : k.range[1]))}` };
+            if (!v?.range)
+                return { problem: `its ${e.name}.sync: has no value to replace` };
+            return { text: `${text.slice(0, v.range[0])}${syncText(sync)}${text.slice(v.range[1])}` };
+        }
+        if (sync === null)
+            return { text };
+        const first = value.items[0].key;
+        const indent = text.slice(lineStart(first.range[0]), first.range[0]);
+        const last = value.items[value.items.length - 1];
+        const lv = last.value;
+        const at = lineEnd(lv?.range ? lv.range[1] : last.key.range[1]);
+        return { text: `${text.slice(0, at)}${at > 0 && text[at - 1] !== "\n" ? "\n" : ""}${indent}sync: ${syncText(sync)}\n${text.slice(at)}` };
+    }
+    if (sync === null)
+        return { text };
+    // a flow map (`orders: {}`, `orders: { columns: [id] }`) or nothing: the entry written again, a comment on its line kept
+    const k = same.pair.key, v = value;
+    // an entry with no value (`orders:   # c`) has an empty scalar placed AT its comment: the comment is read from the key
+    const after = v?.range && v.range[1] > v.range[0] ? v.range[1] : k.range[1];
+    const end = lineEnd(after);
+    const comment = /(\s+#.*)$/.exec(text.slice(after, end).replace(/\n$/, ""))?.[1] ?? "";
+    const indent = text.slice(lineStart(k.range[0]), k.range[0]);
+    const lines = entryText({ ...e, name: same.name, sync }, indent).split("\n");
+    lines[0] = `${lines[0]}${comment}`;
+    return { text: `${text.slice(0, lineStart(k.range[0]))}${lines.join("\n")}${text.slice(end)}` };
+}
 /**
  * The file without its entry for `table`: its lines cut, every other byte as it was; the last entry gone leaves
  * `tables: {}`. Null when no entry names that table.