@mason-data/cli 0.8.0 → 0.9.0

package/dist/src/duck/distinct-key.js

@@ -1,6 +1,9 @@
 /** pg_proc's count(any) */
 const COUNT_ANY = 2147;
 const RTE_RELATION = 0;
+const RTE_SUBQUERY = 1;
+const RTE_CTE = 6;
+const RTE_GROUP = 9;
 /** JoinType: INNER, LEFT, FULL, RIGHT (nodes.h); the planner's SEMI / ANTI never reach a parse tree */
 const JOINS = new Set([0, 1, 2, 3]);
 export const columnKey = (relid, attno) => `${relid}.${attno}`;
@@ -39,22 +42,25 @@ export function distinctCountTables(tree) {
     return counts ? [...relids].sort((a, b) => a - b) : [];
 }
 /**
- * The catalog Postgres's single-column PRIMARY KEYS of `relids` (never another unique index: the module's doc), with
- * each one's collation and the equality operators of its btree family. A primary key is valid, unique and over a plain
- * column; the conditions are written out all the same, so the rule is this query's and not a reader's memory.
+ * The catalog Postgres's PRIMARY KEYS of `relids` (never another unique index: the module's doc), a row per column of
+ * the key, with its collation, the equality operators of its btree family and how many columns the key has. A primary
+ * key is valid, unique and over plain columns; the conditions are written out all the same, so the rule is this
+ * query's and not a reader's memory.
  */
 export function uniqueColumnsSql(relids) {
-    return "SELECT i.indrelid::text, i.indkey[0]::text, i.indcollation[0]::text, coalesce(string_agg(o.amopopr::text, ' ' ORDER BY o.amopopr), '') " +
-        "FROM pg_index i JOIN pg_opclass c ON c.oid = i.indclass[0] JOIN pg_am m ON m.oid = c.opcmethod AND m.amname = 'btree' " +
+    return "SELECT i.indrelid::text, i.indkey[k.at]::text, i.indcollation[k.at]::text, coalesce(string_agg(o.amopopr::text, ' ' ORDER BY o.amopopr), ''), i.indnkeyatts::text " +
+        "FROM pg_index i CROSS JOIN LATERAL generate_series(0, i.indnkeyatts - 1) AS k(at) JOIN pg_opclass c ON c.oid = i.indclass[k.at] JOIN pg_am m ON m.oid = c.opcmethod AND m.amname = 'btree' " +
         "LEFT JOIN pg_amop o ON o.amopfamily = c.opcfamily AND o.amopmethod = c.opcmethod AND o.amopstrategy = 3 " +
-        `WHERE i.indrelid IN (${relids.map((r) => String(Math.trunc(r))).join(", ")}) AND i.indisprimary AND i.indisunique AND i.indisvalid AND i.indnatts = 1 AND i.indnkeyatts = 1 ` +
-        "AND i.indkey[0] > 0 AND i.indpred IS NULL AND i.indexprs IS NULL GROUP BY 1, 2, 3";
+        `WHERE i.indrelid IN (${relids.map((r) => String(Math.trunc(r))).join(", ")}) AND i.indisprimary AND i.indisunique AND i.indisvalid AND i.indnatts = i.indnkeyatts ` +
+        "AND i.indkey[k.at] > 0 AND i.indpred IS NULL AND i.indexprs IS NULL GROUP BY 1, 2, 3, 5";
 }
-/** `uniqueColumnsSql`'s rows, as the decision reads them (a table has one primary key; the merge is for a second row
- *  of one column, which no catalog writes) */
+/** `uniqueColumnsSql`'s rows of a key of one column, as the decision reads them (a table has one primary key; the merge
+ *  is for a second row of one column, which no catalog writes) */
 export function uniqueColumnsOf(rows) {
     const out = new Map();
-    for (const [relid, attno, collation, ops] of rows) {
+    for (const [relid, attno, collation, ops, columns] of rows) {
+        if (columns !== "1")
+            continue;
         const key = columnKey(Number(relid), Number(attno));
         const equalities = new Set((ops ?? "").split(" ").filter((x) => x !== "").map(Number));
         const had = out.get(key);
@@ -67,6 +73,17 @@ export function uniqueColumnsOf(rows) {
     }
     return out;
 }
+/** `uniqueColumnsSql`'s rows, each table's whole key: a key not read whole is no key */
+export function primaryKeysOf(rows) {
+    const out = new Map(), whole = new Map();
+    for (const [relid, attno, collation, ops, columns] of rows) {
+        const key = out.get(Number(relid)) ?? new Map();
+        key.set(Number(attno), { collation: Number(collation ?? 0), equalities: new Set((ops ?? "").split(" ").filter((x) => x !== "").map(Number)) });
+        out.set(Number(relid), key);
+        whole.set(Number(relid), Number(columns));
+    }
+    return new Map([...out].filter(([relid, key]) => key.size === whole.get(relid)));
+}
 /** a VAR of this query's own FROM (varlevelsup 0), bare or under a binary-compatible relabel (a varchar read as text) */
 function ownVar(v) {
     if (!isNode(v))
@@ -118,28 +135,33 @@ function baseTable(rtable, ref) {
     const relid = num(rte.relid);
     return rtindex !== null && relid !== null && relid > 0 ? { rtindex, relid } : null;
 }
+/**
+ * The column of the table at `rtindex` a conjunct equates with a column of `side`, both bare, by the operator and under
+ * the collation `key` says are that column's own; null where it does not.
+ */
+function equated(c, side, rtindex, key) {
+    if (c._node !== "OPEXPR" || list(c.args).length !== 2)
+        return null;
+    const [a, b] = list(c.args).map(ownVar);
+    if (a == null || b == null)
+        return null;
+    for (const [mine, theirs] of [[a, b], [b, a]]) {
+        if (num(theirs.varno) !== rtindex || !side.has(num(mine.varno) ?? -1))
+            continue;
+        const attno = num(theirs.varattno), u = key(attno);
+        const collation = num(c.inputcollid) ?? 0;
+        if (u !== undefined && u.equalities.has(num(c.opno) ?? -1) && (collation === 0 || collation === u.collation))
+            return attno;
+    }
+    return null;
+}
 /**
  * Whether a join's condition matches each row of its `side` with at most ONE row of `other`, a base table: a conjunct
  * that is an equality, by the operator of `other`'s single-column primary key and under its collation, of that
  * column (bare) with a column of `side` (bare).
  */
 function atMostOne(quals, side, other, unique) {
-    return conjuncts(quals).some((c) => {
-        if (c._node !== "OPEXPR" || list(c.args).length !== 2)
-            return false;
-        const [a, b] = list(c.args).map(ownVar);
-        if (a == null || b == null)
-            return false;
-        for (const [mine, theirs] of [[a, b], [b, a]]) {
-            if (num(theirs.varno) !== other.rtindex || !side.has(num(mine.varno) ?? -1))
-                continue;
-            const u = unique.get(columnKey(other.relid, num(theirs.varattno)));
-            const collation = num(c.inputcollid) ?? 0;
-            if (u !== undefined && u.equalities.has(num(c.opno) ?? -1) && (collation === 0 || collation === u.collation))
-                return true;
-        }
-        return false;
-    });
+    return conjuncts(quals).some((c) => equated(c, side, other.rtindex, (attno) => unique.get(columnKey(other.relid, attno))) !== null);
 }
 /**
  * Whether the FROM of `query` holds each row of the table at `rtindex` at most once: its one item, that table or a
@@ -169,6 +191,241 @@ export function notMultiplied(query, rtindex, unique) {
     };
     return down(from[0]);
 }
+/** whether a query is a projection and a WHERE alone: no aggregate, window, set-returning list, subquery expression,
+ *  DISTINCT, grouping, sort, limit, set operation, WITH or row lock, so each of its rows is one row of its FROM */
+export function projectionOnly(q) {
+    for (const f of ["hasAggs", "hasWindowFuncs", "hasTargetSRFs", "hasSubLinks", "hasDistinctOn", "hasRecursive", "hasModifyingCTE", "hasForUpdate"])
+        if (q[f] !== "false")
+            return false;
+    for (const f of ["cteList", "groupClause", "groupingSets", "havingQual", "windowClause", "distinctClause", "sortClause", "limitOffset", "limitCount", "rowMarks", "setOperations"]) {
+        if (q[f] !== null && q[f] !== undefined && !(Array.isArray(q[f]) && q[f].length === 0))
+            return false;
+    }
+    return true;
+}
+/**
+ * THE PLAIN READ: a query that is a projection and a WHERE over ONE table, or over one plain read (a view of a view), so
+ * it holds each row of that table at most once; null for anything else. Whether it may also be read twice is the
+ * reader's (src/duck/late.ts asks too that nothing in it is volatile): this is about its rows alone.
+ */
+export function plainTable(q) {
+    if (!projectionOnly(q) || !isNode(q.jointree))
+        return null;
+    const from = list(q.jointree.fromlist);
+    const rtable = list(q.rtable);
+    if (from.length !== 1)
+        return null;
+    const inner = heldTable(q, rtable, from[0]);
+    if (inner === null)
+        return null;
+    const outputs = list(q.targetList).filter((t) => isNode(t) && t.resjunk !== "true");
+    return { relid: inner.relid, column: (attno) => {
+            const v = ownVar(outputs[attno - 1]?.expr);
+            return v === null ? null : inner.column(num(v.varattno));
+        } };
+}
+/** the table a FROM's RANGETBLREF reads once: a base table, a plain derived read, or a plain read of its query's WITH */
+function heldTable(query, rtable, ref) {
+    const t = baseTable(rtable, ref);
+    if (t !== null)
+        return { relid: t.relid, column: (attno) => attno };
+    if (!isNode(ref) || ref._node !== "RANGETBLREF")
+        return null;
+    const rte = rtable[(num(ref.rtindex) ?? 0) - 1];
+    // a LATERAL read is one per row of what it follows: its rows are no bound by its table's
+    if (!isNode(rte) || rte.lateral === "true")
+        return null;
+    if (num(rte.rtekind) === RTE_SUBQUERY && isNode(rte.subquery))
+        return plainTable(rte.subquery);
+    if (num(rte.rtekind) !== RTE_CTE || (num(rte.ctelevelsup) ?? -1) !== 0)
+        return null;
+    const cte = list(query.cteList).find((c) => isNode(c) && c.ctename === rte.ctename);
+    return isNode(cte) && isNode(cte.ctequery) ? plainTable(cte.ctequery) : null;
+}
+/**
+ * THE FAN-OUT PROOF: whether every join of `query` equates a whole primary key of a table whose rows its side holds at
+ * most once, so the FROM's rows are bounded by the sum of its tables'. A key must be unique IN ITS SIDE, not merely in
+ * its table: `a JOIN b ON a.id = b.a_id JOIN c ON a.id = c.a_id` equates a's key twice, and an `a` of a thousand b's and
+ * a thousand c's is a million rows, because the first join repeats `a`. So a join passes on, as held at most once, only
+ * the tables of a side whose every row it matches at most once.
+ *
+ * Each key column in an equality, by its btree family's operator and under its collation, with a bare column of the
+ * other side (`equated`); conjuncts of the join's top-level AND only, so an OR proves nothing and any other conjunct
+ * only filters. INNER, LEFT, RIGHT and FULL alike: a side's unmatched row is one row, and its NULL-extended key
+ * equals nothing. A comma's items are inner joins, left to right, on the WHERE's conjuncts. Everything else (a cross
+ * join, a key in part, a partial or UNIQUE index, no key, a key equated with an expression or a constant, a FROM item
+ * neither a base table nor a plain read of one, a JOIN's condition written in the WHERE) proves nothing.
+ *
+ * What it decides is what a LIMIT with no ORDER BY costs and whether its pages are stable, never the rows' values, so
+ * a key stale in the catalog (the source's as of the start) costs time, not an answer.
+ */
+export function fanOutBounded(query, keys) {
+    const jt = query.jointree;
+    if (!isNode(jt))
+        return false;
+    const rtable = list(query.rtable);
+    const joined = (left, right, quals, indexes) => {
+        if (left === null || right === null)
+            return null;
+        const matchedOnce = (s, by) => [...s.once].some(([rtindex, table]) => {
+            const key = keys.get(table.relid);
+            if (key === undefined)
+                return false;
+            const keyColumn = (attno) => { const c = table.column(attno); return c === null ? undefined : key.get(c); };
+            const met = new Set(quals.map((c) => { const at = equated(c, by.indexes, rtindex, keyColumn); return at === null ? null : table.column(at); }));
+            return [...key.keys()].every((attno) => met.has(attno));
+        });
+        const leftMatchesOne = matchedOnce(right, left), rightMatchesOne = matchedOnce(left, right);
+        if (!leftMatchesOne && !rightMatchesOne)
+            return null;
+        return { indexes, once: new Map([...(leftMatchesOne ? left.once : []), ...(rightMatchesOne ? right.once : [])]) };
+    };
+    const side = (n) => {
+        if (isNode(n) && n._node === "JOINEXPR" && JOINS.has(num(n.jointype) ?? -1) && isNode(n.larg) && isNode(n.rarg))
+            return joined(side(n.larg), side(n.rarg), conjuncts(n.quals), indexesOf(n));
+        const t = heldTable(query, rtable, n);
+        const rtindex = isNode(n) ? num(n.rtindex) : null;
+        return t === null || rtindex === null ? null : { indexes: new Set([rtindex]), once: new Map([[rtindex, t]]) };
+    };
+    const where = conjuncts(jt.quals);
+    const items = list(jt.fromlist).map(side);
+    if (items.length === 0)
+        return false;
+    return items.reduce((l, r) => joined(l, r, where, new Set([...l?.indexes ?? [], ...r?.indexes ?? []]))) !== null;
+}
+/** the base tables a query reads, at any depth (a wrapper's LIMIT stands over the join inside it) */
+function tablesUnder(q) {
+    const out = [];
+    const visit = (v) => {
+        if (Array.isArray(v)) {
+            v.forEach(visit);
+            return;
+        }
+        if (!isNode(v))
+            return;
+        if (v._node === "RANGETBLENTRY" && num(v.rtekind) === RTE_RELATION) {
+            const r = num(v.relid) ?? 0;
+            if (r > 0)
+                out.push(r);
+        }
+        for (const [k, x] of Object.entries(v))
+            if (k !== "_node")
+                visit(x);
+    };
+    visit(q);
+    return out;
+}
+/** The base tables `fanOutBounded` may be asked about: those under a query with a LIMIT and no sort, at least two of them
+ *  (none: the plan asks the catalog nothing). */
+export function fanOutTables(tree) {
+    const relids = new Set();
+    const visit = (v) => {
+        if (Array.isArray(v)) {
+            v.forEach(visit);
+            return;
+        }
+        if (!isNode(v))
+            return;
+        if (v._node === "QUERY" && isNode(v.limitCount) && list(v.sortClause).length === 0) {
+            const tables = tablesUnder(v);
+            if (tables.length >= 2)
+                for (const r of tables)
+                    relids.add(r);
+        }
+        for (const [k, x] of Object.entries(v))
+            if (k !== "_node")
+                visit(x);
+    };
+    visit(tree);
+    return [...relids].sort((a, b) => a - b);
+}
+/** a grouped expression's collation where it is a bare column (a VAR's, or a binary-compatible relabel's own) */
+const collationOf = (v) => !isNode(v) ? 0 : v._node === "RELABELTYPE" ? num(v.resultcollid) ?? 0 : num(v.varcollid) ?? 0;
+/**
+ * THE KEY-FIRST GROUPING PROOF (issue 134, step 6; src/duck/late.ts `keyFirst`): the base tables of `query` whose WHOLE
+ * primary key it groups by, by rtindex, each with its key's columns by attno. A group then holds one row of that table
+ * (or none, where an outer join supplied none), so every column of it is decided by the group, and the row can be read
+ * by its row number after the limit. Each key column grouped bare (a binary-compatible relabel allowed), by its btree
+ * family's equality and under its collation; a grouping set proves nothing (a key reads NULL in its total rows). The
+ * key is the primary key, never a UNIQUE index (the module's doc): the copy is written by it. A key the copy holds
+ * twice for a moment (a change applied in pieces) is grouped as written, never by a row number, so no group is split.
+ * A SELECT DISTINCT that is a grouping on its list (`distinctGrouping`) is read as its GROUP BY: its distinctClause.
+ */
+export function keyGrouped(query, keys) {
+    const out = new Map();
+    if (list(query.groupingSets).length > 0)
+        return out;
+    const clause = list(query.groupClause).length > 0 ? list(query.groupClause) : distinctGrouping(query) ? list(query.distinctClause) : [];
+    const targets = list(query.targetList).filter(isNode);
+    const rtable = list(query.rtable);
+    const grouped = new Map();
+    // a grouped output is a VAR of the query's RTE_GROUP entry (Postgres 17+): the expression grouped is its groupexprs'
+    const grouped1 = (e) => {
+        const v = ownVar(e), rte = v === null ? undefined : rtable[(num(v.varno) ?? 0) - 1];
+        return isNode(rte) && num(rte.rtekind) === RTE_GROUP ? list(rte.groupexprs)[(num(v.varattno) ?? 0) - 1] : e;
+    };
+    for (const g of clause) {
+        if (!isNode(g))
+            continue;
+        const t = targets.find((x) => x.ressortgroupref === g.tleSortGroupRef);
+        const expr = grouped1(t?.expr);
+        const v = ownVar(expr);
+        const rtindex = num(v?.varno), attno = num(v?.varattno);
+        if (v === null || rtindex === null || attno === null)
+            continue;
+        const table = baseTable(rtable, { _node: "RANGETBLREF", rtindex: String(rtindex) });
+        const key = table === null ? undefined : keys.get(table.relid)?.get(attno);
+        const collation = collationOf(expr);
+        if (key === undefined || !key.equalities.has(num(g.eqop) ?? -1) || (collation !== 0 && collation !== key.collation))
+            continue;
+        grouped.set(rtindex, new Set([...grouped.get(rtindex) ?? [], attno]));
+    }
+    for (const [rtindex, attnos] of grouped) {
+        const table = baseTable(rtable, { _node: "RANGETBLREF", rtindex: String(rtindex) });
+        const key = [...keys.get(table.relid).keys()];
+        if (key.every((a) => attnos.has(a)))
+            out.set(rtindex, key);
+    }
+    return out;
+}
+/**
+ * Whether a query is a SELECT DISTINCT that is a grouping on its list (issue 134, step 6, task D): a DISTINCT, not a
+ * DISTINCT ON (which keeps one row of each set, a different thing), with no GROUP BY beside it, and nothing in its list
+ * computed before the DISTINCT that a grouping would compute after it: no aggregate (a DISTINCT over one row), window
+ * function, set-returning function or subquery. Such a DISTINCT's rows are its GROUP BY's, its list's equalities the
+ * same, so `keyGrouped` reads its distinctClause as it reads a groupClause.
+ */
+export function distinctGrouping(q) {
+    if (list(q.distinctClause).length === 0 || list(q.groupClause).length > 0 || q.hasDistinctOn !== "false")
+        return false;
+    return ["hasAggs", "hasWindowFuncs", "hasTargetSRFs", "hasSubLinks"].every((f) => q[f] === "false");
+}
+/** The base tables `keyGrouped` may be asked about: those of a query that groups, or is a DISTINCT, and has a LIMIT
+ *  (none: the plan asks the catalog nothing) */
+export function groupedLimitTables(tree) {
+    const relids = new Set();
+    const visit = (v) => {
+        if (Array.isArray(v)) {
+            v.forEach(visit);
+            return;
+        }
+        if (!isNode(v))
+            return;
+        if (v._node === "QUERY" && isNode(v.limitCount) && (list(v.groupClause).length > 0 || distinctGrouping(v))) {
+            for (const e of list(v.rtable))
+                if (isNode(e) && num(e.rtekind) === RTE_RELATION) {
+                    const r = num(e.relid) ?? 0;
+                    if (r > 0)
+                        relids.add(r);
+                }
+        }
+        for (const [k, x] of Object.entries(v))
+            if (k !== "_node")
+                visit(x);
+    };
+    visit(tree);
+    return [...relids].sort((a, b) => a - b);
+}
 /** Whether this AGGREF of `query` is a count(DISTINCT x) whose DISTINCT changes nothing (the module's doc). */
 function distinctOfKey(query, a, unique) {
     if (!isDistinctCount(a) || (num(a.agglevelsup) ?? 0) !== 0 || a.aggkind !== "n" || a.aggstar === "true" || list(a.aggorder).length > 0)

package/dist/src/duck/events.js

@@ -1,19 +1,46 @@
+/**
+ * THE SYNC'S EVENTS (`_mason.sync_events`): a copy failed, a poll failed, a catch-up failed, the stream failed or
+ * refused a table, the database could not be reached — and the notable good news beside them: a copy finished, a
+ * table current again after a failure. Before this a sync failure was kept only as a table's CURRENT error (the next
+ * success wiped it) and, before that one, only in the log file: `mason status` could not say what went wrong an hour ago.
+ *
+ * Kept as the request log is kept (src/duck/history.ts, the frame both share): rows wait here for at most FLUSH_MS and go in ONE INSERT, on a
+ * connection of the events' own — never the sync's writer turn (`onWriter`), which a swap or a stream batch may hold
+ * for seconds; a read is on a third connection and sees committed batches only; a write that fails is said once with
+ * the events it cost, and again when a write succeeds after it, and it NEVER fails the sync that recorded it (the
+ * sync's own state and log line are written either way). Kept `MASON_HISTORY_DAYS` days.
+ *
+ * A failure repeated word for word is ONE event (`REPEAT_MS`): a database away fails every pass, every few seconds,
+ * and a list of the same line buries the one before it. It is written again once it was over in between, or an hour
+ * later: a table's failure is over when the table is current again (`recovered`) or copied anew (`copied`); a failure
+ * of the whole sync (`table: null`: the stream, the database) when the sync says it is (`recovered` of no table: the
+ * stream applied a batch again, the database attached again), which ends every whole-sync failure's repeat at once —
+ * an extra event at worst, never a failure dropped. While it lasts, the table's own `error` (mason status) says it.
+ *
+ * The run's RESTARTS are not here: the supervisor that records them is a process of its own, which cannot open the
+ * file DuckDB holds, so it writes `.mason/restarts.json` (src/cli/supervise.ts) and the reader merges them
+ * (`withRestarts`).
+ *
+ * The Postgres catalog never lists `_mason`, so a tool connected to the listener cannot read the events.
+ */
+import { maskPasswordKeyword } from "../lib/secret-mask.js";
 import { MASON_SCHEMA } from "./store.js";
-import { Batches, History, Streak } from "./history.js";
+import { Batches, History, Streak, cutoff } from "./history.js";
 export const EVENTS_TABLE = `${MASON_SCHEMA}.sync_events`;
 export const EVENTS_DDL = `CREATE TABLE IF NOT EXISTS ${EVENTS_TABLE} (happened_at TIMESTAMPTZ, seq BIGINT, kind VARCHAR, table_name VARCHAR, message VARCHAR)`;
 /**
  * WHAT AN EVENT IS, one word each: `copy_failed`, `poll_failed`, `catch_up_failed` (the one poll that brings a new copy
  * current), `stream_failed` (the stream stopped, or refused a table: `table` says which), `source_unreachable` (the
- * database could not be attached), `restart` (the supervisor started the run again after it died) — each a failure;
- * `copied` (a copy finished and was swapped in), `recovered` (a table whose poll, catch-up or stream failed is
+ * database could not be attached), `generated_failed` (a generated table's re-run failed, and it serves what it held),
+ * `restart` (the supervisor started the run again after it died) — each a failure;
+ * `copied` (a copy finished and was swapped in), `recovered` (a table whose poll, catch-up, stream or re-run failed is
  * current again; of no table, the stream applying changes or the database attached again after it failed) and
  * `synced` (a sync asked by hand — mason sync, the page's Sync now — and what it did), and `sync_changed` (a table's
  * `sync:` in the sources file changed while it is copied: from what, to what, and how its copy is caught up). A scheduled poll that succeeds
  * is no event: one every few seconds would bury the failures; its last result is the table's last sync.
  */
-export const EVENT_KINDS = ["copy_failed", "poll_failed", "catch_up_failed", "stream_failed", "source_unreachable", "restart", "copied", "recovered", "synced", "sync_changed"];
-const FAILURES = new Set(["copy_failed", "poll_failed", "catch_up_failed", "stream_failed", "source_unreachable", "restart"]);
+export const EVENT_KINDS = ["copy_failed", "poll_failed", "catch_up_failed", "stream_failed", "source_unreachable", "generated_failed", "restart", "copied", "recovered", "synced", "sync_changed"];
+const FAILURES = new Set(["copy_failed", "poll_failed", "catch_up_failed", "stream_failed", "source_unreachable", "generated_failed", "restart"]);
 /** whether an event of this kind is a failure: the one place that says it (the reader's filter, `failed`) */
 export const isFailure = (kind) => FAILURES.has(kind);
 const FLUSH_MS = 1000;
@@ -90,9 +117,11 @@ export class SyncEvents {
         }
         this.lost.succeeded((n) => this.log(`sync events: recording again; ${n} event(s) were not recorded (the log has them)`));
     }
-    /** Deletes the events older than the retention; the count is said when there were any. */
+    /** Deletes the events older than the retention; the count is said when there were any. The retention is counted back
+     *  from the module's ONE clock (`now`, which stamps each event), never the engine's `now()`: two clocks disagree
+     *  whenever the clock is given, and a trim by the other deleted what the given clock still kept. */
     async trim() {
-        const r = await this.history.writer.runAndReadAll(`DELETE FROM ${EVENTS_TABLE} WHERE happened_at < now() - INTERVAL (${this.days}) DAY`);
+        const r = await this.history.writer.runAndReadAll(`DELETE FROM ${EVENTS_TABLE} WHERE happened_at < $1::TIMESTAMPTZ`, [cutoff(this.now(), this.days)]);
         const n = Number(r.getRowsJS()[0]?.[0] ?? 0);
         if (n > 0)
             this.log(`sync events: deleted ${n} event(s) older than ${this.days} days (MASON_HISTORY_DAYS)`);
@@ -120,7 +149,9 @@ export class SyncEvents {
         this.history.closeConnections();
     }
 }
-const eventOf = (at, kind, table, message) => ({ at, kind, failed: isFailure(kind), table, message });
+/** an event as read back, any `password=` value in its message masked (src/lib/secret-mask.ts `maskPasswordKeyword`,
+ *  the exact form an earlier Mason leaked, issue 599: the sync scrubs those rows at open, and this holds should one stay) */
+const eventOf = (at, kind, table, message) => ({ at, kind, failed: isFailure(kind), table, message: maskPasswordKeyword(message) });
 /**
  * THE EVENTS WITH THE RUN'S RESTARTS (src/cli/supervise.ts `.mason/restarts.json`), as ONE list newest last, asked as
  * `recent` is (the newest `limit`, since, the failures only — a restart is one), within the retention as the events are.

package/dist/src/duck/executor-worker.js

@@ -14,7 +14,7 @@ import { parentPort, workerData } from "node:worker_threads";
 import { openDuck, SessionRunner } from "./store.js";
 import { FUNCTION_CALLS, FUNCTION_ROWS, registerLibcOrder, useCancelTable } from "./functions.js";
 import { stopPgEval } from "./pg-eval.js";
-import { StatementError } from "../api/postgres/server.js";
+import { StatementError } from "./statement.js";
 import { STOPPING, wireError } from "./executor.js";
 const { cancels } = workerData;
 useCancelTable(cancels);
@@ -134,7 +134,7 @@ async function answer(m) {
             return undefined;
         }
         case "libc":
-            registerLibcOrder(need(conns, 0, "connection"), m.order);
+            registerLibcOrder(need(conns, 0, "connection"), String(m.collate), m.order, m.database === true);
             return { value: null };
         case "calls": return { value: Object.fromEntries(FUNCTION_CALLS) };
         case "function rows": return { value: Object.fromEntries(FUNCTION_ROWS) };

package/dist/src/duck/executor.js

@@ -35,7 +35,7 @@ import { Worker } from "node:worker_threads";
 import { fileURLToPath } from "node:url";
 import * as duck from "@duckdb/node-api";
 import { runnable } from "../lib/runnable.js";
-import { StatementError } from "../api/postgres/server.js";
+import { StatementError } from "./statement.js";
 import { EncodeError } from "../api/postgres/encode.js";
 import { NumericError } from "../lib/pg-numeric.js";
 import { beats, CANCEL_SLOTS, cancelStatement, cancelTable, resetSlot } from "./cancel.js";
@@ -116,7 +116,7 @@ export function wireError(e) {
     if (e instanceof EncodeError)
         return { kind: "encode", message: e.message, code: e.code, gap: e.gap };
     if (e instanceof NumericError)
-        return { kind: "numeric", message: e.message, sqlstate: e.sqlstate, gap: e.gap };
+        return { kind: "numeric", message: e.message, sqlstate: e.sqlstate, gap: e.gap, detail: e.detail };
     return { kind: "error", message: e instanceof Error ? e.message : String(e), name: e instanceof Error ? e.name : "Error" };
 }
 /** the worker is gone: what every request in flight, and every later one, is answered. `stopped`: by the run's own stop
@@ -135,7 +135,7 @@ const errorOf = (e) => {
     if (e.kind === "encode")
         return new EncodeError(e.message, e.code, e.gap);
     if (e.kind === "numeric")
-        return new NumericError(e.message, e.sqlstate, e.gap);
+        return new NumericError(e.message, e.sqlstate, e.gap, e.detail);
     const err = new Error(e.message);
     err.name = e.name;
     return err;
@@ -368,8 +368,9 @@ export class Executor {
     /** a connection of the instance */
     async connect() { return new RemoteConnection(this, await this.call({ op: "connect" })); }
     writer() { return new RemoteConnection(this, 0); }
-    /** the libc order's two functions (src/duck/functions.ts registerLibcOrder), registered on the worker */
-    async registerLibcOrder(order) { await this.call({ op: "libc", order }); }
+    /** a libc order's functions (src/duck/functions.ts registerLibcOrder), registered on the worker: the key under
+     *  `collate`, and jsonb's order under the database's (`database`) */
+    async registerLibcOrder(collate, order, database) { await this.call({ op: "libc", collate, order, database }); }
     /** how many vectors each Mason function was called with so far (src/duck/functions.ts FUNCTION_CALLS: the worker's
      *  count, the functions being its) — what a test or a measurement reads to know whether a statement reached one */
     async functionCalls() { return new Map(Object.entries(await this.call({ op: "calls" }))); }

package/dist/src/duck/explain.js

@@ -113,6 +113,8 @@ export function keptText(f, now = Date.now(), rows = true) {
     const k = f.kept;
     if (k === undefined)
         return UNKNOWN;
+    if (k.by === "generated")
+        return k.text;
     if (k.by === "poll") {
         const synced = k.syncedAt === null ? "never synced" : `last synced ${agoText(k.syncedAt, now)} (${utc(k.syncedAt)})`;
         if (k.pending !== undefined)
@@ -144,6 +146,8 @@ export function keptCurrentBy(f) {
         return k.pending === undefined && k.sync.mode === "poll" ? "polling" : null;
     if (k.by === "recopy")
         return "copying again";
+    if (k.by === "generated")
+        return null;
     return k.state === "streamed" && k.notRunning === null ? "logical replication" : null;
 }
 /** how a copy is kept current, as FORMAT JSON's keys say it (`keptText`'s facts) */