@mason-data/cli 0.6.2 → 0.8.0

package/dist/src/duck/grants.js

@@ -0,0 +1,176 @@
+/**
+ * ROLES: WHAT A USER MAY READ, HELD ON THE ANALYSED TREE (GitHub issue 30, stage 1: table and column grants per role).
+ * A role is an entry of the project's roles file (`mason/roles.yml`, src/cli/roles.ts): for each copied table, every
+ * column (`{}`), `columns:` or `exclude:`, the sources file's own words — or `tables: all`, every copied table and column.
+ * A `mason user` carries a role (src/duck/users.ts `MasonUser.role`); a user with none is held to the file's
+ * `default:` role, and reads nothing when the file names none (stage 1b: the FILE decides, never the code); with no
+ * roles file at all, every user reads every copied table. A token (no user) reads every copied table.
+ *
+ * ENFORCED ON THE TREE POSTGRES ANALYSED, NEVER ON SQL TEXT: Postgres puts one `RTEPERMISSIONINFO` per relation in the
+ * query tree it analyses (src/duck/analyze.ts), with `selectedCols` as its own bitmap of the columns the statement
+ * reads — a column read inside a CTE, a subquery, a join's USING, a window, a WHERE or a CASE is in it; `SELECT *` lists
+ * every column; `count(*)` reads none; a whole-row reference (`SELECT t FROM t`) is the bit for attnum 0, which means
+ * every column. This is the information Postgres's own executor checks privileges with (ExecCheckPermissions), so an
+ * alias, a subquery or a view cannot get around it: a view is expanded by Mason itself before the check
+ * (src/duck/views.ts), so the check sees the tables it reads — stricter than Postgres, where a grant on the view
+ * suffices, and said so in docs/POSTGRES-API.md.
+ *
+ * THE REFUSAL IS POSTGRES'S: 42501 `permission denied for table orders`, the words Postgres 18 answers whether the
+ * table or one of its columns is missing from the grants (its executor names the table only), so a tool handles it as
+ * it handles Postgres's. The DETAIL names the role and the first hidden column, the HINT what the role may read. The
+ * rules are Postgres's too: `count(*)` over a table is answered when the role has any column of it; `SELECT *` over a
+ * table the role has only part of is refused, never narrowed (a narrowed `*` would change the result's columns); a
+ * system column (xmin, ctid) needs the whole table.
+ */
+import { StatementError } from "../api/postgres/server.js";
+import { catalogRows } from "./analyze.js";
+/** no roles file: every session reads every copied table, as before; one object, so the listener sees no change */
+export const NO_ROLES = { file: null, grants: new Map(), problem: null, default: null };
+/**
+ * THE CATALOG'S ROLE OF A USER THE FILE HOLDS TO NOTHING (no role of its own, no `default:`): a Postgres role with no
+ * grant, so its catalog requests answer as Postgres answers a login with none — information_schema lists none of the
+ * copied tables or views, every `has_*_privilege` is false, `set_config` is revoked (while any role exists; a session's
+ * own `set_config` never reaches the catalog, src/api/postgres/translate.ts setConfig).
+ * Named outside the file's namespace: `ROLE_NAME_RE` (src/cli/roles.ts) forbids a leading underscore, so no role of the
+ * file can ever be it.
+ */
+export const NO_ROLE_CATALOG = "__mason_no_role";
+/** The roles the catalog Postgres holds (src/api/postgres/catalog.ts catalogRoles): the file's, and — whenever a roles
+ *  file exists — `NO_ROLE_CATALOG`, a role with no table, for a user the file holds to nothing. */
+export function catalogRolesOf(live) {
+    if (live.file === null)
+        return live.grants;
+    return new Map([...live.grants, [NO_ROLE_CATALOG, { name: NO_ROLE_CATALOG, file: live.file, tables: new Map(), all: false }]]);
+}
+/** `FirstLowInvalidHeapAttributeNumber` is -7: a column's bit in `selectedCols` is its attnum + 7 */
+const ATTNUM_OFFSET = 7;
+/** ACL_SELECT in `requiredPerms` */
+const ACL_SELECT = 2;
+const isNode = (v) => v !== null && v !== undefined && typeof v === "object" && !Array.isArray(v);
+/** Every RTEPERMISSIONINFO of the tree that asks SELECT, with its bitmap read as attnums. */
+export function tableReads(tree) {
+    const out = [];
+    const visit = (v) => {
+        if (Array.isArray(v)) {
+            for (const x of v)
+                visit(x);
+            return;
+        }
+        if (!isNode(v))
+            return;
+        if (v._node === "RTEPERMISSIONINFO") {
+            const perms = Number(v.requiredPerms ?? 0);
+            const relid = Number(v.relid ?? 0);
+            if ((perms & ACL_SELECT) !== 0 && relid > 0) {
+                // a bitmapset prints as `(b 8 9)`: the `b` first, then each member
+                const bits = Array.isArray(v.selectedCols) ? v.selectedCols.filter((x) => typeof x === "string" && /^\d+$/.test(x)).map(Number) : [];
+                const read = { relid, attnums: [], wholeRow: false, system: false };
+                for (const b of bits) {
+                    const attnum = b - ATTNUM_OFFSET;
+                    if (attnum === 0)
+                        read.wholeRow = true;
+                    else if (attnum < 0)
+                        read.system = true;
+                    else
+                        read.attnums.push(attnum);
+                }
+                out.push(read);
+            }
+        }
+        for (const [k, x] of Object.entries(v))
+            if (k !== "_node")
+                visit(x);
+    };
+    visit(tree);
+    return out;
+}
+/**
+ * The reads named through the catalog Postgres's own pg_class and pg_attribute (dropped columns left out, as Postgres
+ * leaves them): the copied tables only (relkind r) — a view is never here, since Mason expands it before the plan is
+ * made (src/duck/views.ts), and a catalog relation is not a copy.
+ */
+export async function resolveReads(catalog, reads) {
+    if (reads.length === 0)
+        return [];
+    const relids = [...new Set(reads.map((r) => Math.trunc(r.relid)))];
+    const rows = await catalogRows(catalog, `SELECT c.oid::text, n.nspname, c.relname, a.attnum::text, a.attname FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace ` +
+        `LEFT JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped WHERE c.oid IN (${relids.join(", ")}) AND c.relkind = 'r' ` +
+        // the copied tables alone: pg_class and the rest of pg_catalog are relkind 'r' too, and PUBLIC's on Postgres (a review finding of #35)
+        `AND n.nspname NOT IN ('pg_catalog', 'information_schema') ORDER BY c.oid, a.attnum`);
+    const tables = new Map();
+    for (const [oid, schema, name, attnum, attname] of rows) {
+        const t = tables.get(Number(oid)) ?? { table: `${schema}.${name}`, columns: new Map() };
+        if (attnum !== null && attnum !== undefined && attname !== null && attname !== undefined)
+            t.columns.set(Number(attnum), attname);
+        tables.set(Number(oid), t);
+    }
+    const out = [];
+    for (const r of reads) {
+        const t = tables.get(r.relid);
+        if (t === undefined)
+            continue;
+        const all = [...t.columns.entries()].sort((a, b) => a[0] - b[0]).map(([, n]) => n);
+        const named = r.wholeRow ? all : [...r.attnums].sort((a, b) => a - b).map((n) => t.columns.get(n)).filter((n) => n !== undefined);
+        out.push({ table: t.table, columns: named, all, system: r.system });
+    }
+    return out;
+}
+/** a table as a tool writes it: its name alone in public, else schema.table (the sources file's own spelling) */
+const shown = (table) => table.startsWith("public.") ? table.slice("public.".length) : table;
+/** The 42501 of a read the role does not cover, Postgres's words first; null when the role covers every read. */
+export function grantRefusal(reads, role) {
+    if (role.noRole !== undefined)
+        return reads.length === 0 ? null : noRoleRefusal(reads[0].table, role.file, role.noRole);
+    const tablesOf = () => {
+        if (role.all)
+            return "every synced table (tables: all)";
+        const names = [...role.tables.keys()].map(shown);
+        return names.length ? names.join(", ") : "none";
+    };
+    for (const r of reads) {
+        const g = role.tables.get(r.table);
+        const denied = (detail, hint) => new StatementError({ code: "42501", message: `permission denied for table ${r.table.replace(/^[^.]+\./, "")}`, detail, hint });
+        if (g === undefined) {
+            return denied(`the role ${role.name} has no grant on ${shown(r.table)}`, `the role ${role.name} reads: ${tablesOf()} (${role.file} names what each role reads; Mason gap roles)`);
+        }
+        if (g.all)
+            continue;
+        const may = `the role ${role.name} reads these columns of ${shown(r.table)}: ${g.columns.join(", ")} (${role.file} names what each role reads; Mason gap roles)`;
+        if (r.system)
+            return denied(`the role ${role.name} may read columns of ${shown(r.table)}, and a system column needs the whole table`, may);
+        const hidden = r.columns.find((c) => !g.columns.includes(c));
+        if (hidden !== undefined)
+            return denied(`the role ${role.name} may not read column ${hidden} of ${shown(r.table)}`, may);
+        // no column read (count(*)): any column of the table is enough, as Postgres rules it, and a grant names at least one
+    }
+    return null;
+}
+/**
+ * THE 42501 OF A USER WITH NO ROLE UNDER A FILE THAT NAMES NO DEFAULT (stage 1b): Postgres's message for the first
+ * table read, the reason in the DETAIL, and both ways out as the exact lines in the HINT — the file's `default:`, or
+ * the user made again with a role. `file`: the roles file as messages name it.
+ */
+export function noRoleRefusal(table, file, who) {
+    return new StatementError({ code: "42501", message: `permission denied for table ${table.replace(/^[^.]+\./, "")}`,
+        detail: `the user "${who.user}" has no role and ${file} names no default`, hint: noRoleWayOut(who.user, file, who.example) });
+}
+/** The two ways out for a user with no role under a file with no default, as exact lines: ONE wording, for the
+ *  refusal's HINT, `mason start`, `mason user list` and `mason user add`. */
+export const noRoleWayOut = (user, file, example) => `write default: ${example} at the top of ${file}, or mason user remove ${user} and mason user add ${user} --role ${example}`;
+/** A role of the file to name in a way out: one of `tables: all` where there is one (the role today's access is), else
+ *  the first, or a placeholder where it names none. */
+export const exampleRole = (live) => [...live.grants.values()].find((g) => g.all)?.name ?? [...live.grants.keys()][0] ?? "<a role>";
+/**
+ * The statement's table reads held to the role: the tree's reads named through the catalog, then refused where the role
+ * does not cover one. Called by the plan (src/duck/plan.ts) after Postgres's analysis and the views' expansion, before
+ * the rewrite — so Postgres's own refusal (an unknown column) comes first, and a role's before a Mason gap.
+ */
+export async function enforceGrants(catalog, tree, role) {
+    const reads = await resolveReads(catalog, tableReads(tree));
+    if (role !== null) {
+        const refused = grantRefusal(reads, role);
+        if (refused !== null)
+            throw refused;
+    }
+    return reads;
+}

package/dist/src/duck/history.js

@@ -0,0 +1,123 @@
+import { MASON_SCHEMA } from "./store.js";
+/** How many days every history table keeps (the request log, the sync's events, the usage samples, the copies'
+ *  sizes) unless MASON_HISTORY_DAYS says. */
+export const DEFAULT_HISTORY_DAYS = 7;
+const KEPT = "the request log, the sync's events, the usage samples and the synced tables' sizes";
+/** The days MASON_HISTORY_DAYS keeps, or what is wrong with the text. Unset or empty: the default. */
+export function historyDays(text) {
+    if (text === undefined || text.trim() === "")
+        return DEFAULT_HISTORY_DAYS;
+    const days = Number(text);
+    if (!Number.isInteger(days) || days < 1) {
+        return { message: `MASON_HISTORY_DAYS=${JSON.stringify(text)} is not a number of days`,
+            why: `it is how many days Mason keeps its history (${KEPT}): a whole number, 1 or more`,
+            fix: `write MASON_HISTORY_DAYS=${DEFAULT_HISTORY_DAYS} (the default, as when it is unset) or another whole number of days` };
+    }
+    return days;
+}
+/**
+ * THE HISTORY'S DAYS, AS THE ENVIRONMENT SETS THEM — read by this one function at every entrance (`mason serve`'s
+ * settings report, `mason start` before it spawns, and the run before anything opens), so a wrong setting is refused
+ * before the copy is touched, in the Failure shape. `MASON_REQUEST_LOG_DAYS` SET IS REFUSED, never run on the default
+ * (CLAUDE.md: a renamed setting refuses the start, naming the line to write): it kept the request log alone once, and
+ * the days kept are every history table's now, so it is MASON_HISTORY_DAYS. Then MASON_HISTORY_DAYS must be a number
+ * of days (`historyDays`).
+ */
+export function historyDaysSetting(env) {
+    const v = env.MASON_REQUEST_LOG_DAYS;
+    if (v !== undefined && v !== "") {
+        return { message: `MASON_REQUEST_LOG_DAYS=${v} is no longer read`,
+            why: `it was renamed MASON_HISTORY_DAYS, since the days it keeps are every history's (${KEPT})`,
+            fix: `write MASON_HISTORY_DAYS=${v.trim()} instead, then unset MASON_REQUEST_LOG_DAYS` };
+    }
+    return historyDays(env.MASON_HISTORY_DAYS);
+}
+/** what is wrong, as one line of `mason serve`'s settings report */
+export function historyDaysWrongText(w) {
+    return `${w.message}: ${w.why} — ${w.fix}`;
+}
+/** how often the rows past the retention are deleted (and once at open) */
+export const TRIM_EVERY_MS = 60 * 60 * 1000;
+/** A history table's connections and its hourly trim. */
+export class History {
+    writer;
+    reader;
+    trimTimer = null;
+    constructor(writer, reader) { this.writer = writer; this.reader = reader; }
+    /** The writer connected, `_mason` and each of `ddl` run on it in order, then the reader connected. */
+    static async open(store, ddl) {
+        const writer = await store.instance.connect();
+        await writer.run(`CREATE SCHEMA IF NOT EXISTS ${MASON_SCHEMA}`);
+        for (const d of ddl)
+            await writer.run(d);
+        const reader = await store.instance.connect();
+        return new History(writer, reader);
+    }
+    /** `trim` now (its failure is the open's), then every TRIM_EVERY_MS, an hourly one that fails said by `failed`. */
+    async keepTrimmed(trim, failed) {
+        await trim();
+        this.trimTimer = setInterval(() => { trim().catch((e) => failed(e)); }, TRIM_EVERY_MS);
+        this.trimTimer.unref();
+    }
+    /** No more hourly trims (the first step of a close). */
+    stopTrimming() {
+        if (this.trimTimer !== null) {
+            clearInterval(this.trimTimer);
+            this.trimTimer = null;
+        }
+    }
+    /** Both connections closed (the last step of a close). */
+    closeConnections() {
+        this.writer.closeSync();
+        this.reader.closeSync();
+    }
+}
+/** A streak of failures, said once: `failed` says the first of a streak and counts what each cost; `succeeded` says,
+ *  once, what the streak cost, and ends it. */
+export class Streak {
+    lost = 0;
+    failed(cost, sayFirst) {
+        if (this.lost === 0)
+            sayFirst();
+        this.lost += cost;
+    }
+    succeeded(sayEnd) {
+        if (this.lost > 0) {
+            sayEnd(this.lost);
+            this.lost = 0;
+        }
+    }
+}
+/** Rows written in batches: `push` one; a batch is written by `write` once `rows` are waiting or after `ms`, one batch
+ *  at a time. `write` must not throw: a failure is its own to say (a Streak). */
+export class Batches {
+    buffer = [];
+    timer = null;
+    /** the write in flight: one at a time, so batches commit in the order they were taken */
+    flushing = Promise.resolve();
+    rows;
+    ms;
+    write;
+    constructor(o) { this.rows = o.rows; this.ms = o.ms; this.write = o.write; }
+    push(row) {
+        this.buffer.push(row);
+        if (this.buffer.length >= this.rows) {
+            void this.flush();
+            return;
+        }
+        this.timer ??= setTimeout(() => { void this.flush(); }, this.ms);
+    }
+    /** Writes what is waiting, after any write in flight. Resolves whether it was written or not (`write` says a failure). */
+    flush() {
+        if (this.timer !== null) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+        const rows = this.buffer;
+        this.buffer = [];
+        if (rows.length === 0)
+            return this.flushing;
+        this.flushing = this.flushing.then(() => this.write(rows));
+        return this.flushing;
+    }
+}

package/dist/src/duck/jsonb-order.js

@@ -0,0 +1,175 @@
+/**
+ * JSONB'S ORDER AND EQUALITY AS BYTES (a BI tool sorts any column a user clicks: `ORDER BY "e"."tags" DESC` over a jsonb
+ * column was refused until this). Postgres orders and equates jsonb by `compareJsonbContainers` (jsonb_util.c), which
+ * walks both values token by token and stops at the first difference:
+ *  - by TYPE first, at any depth: null < string < number < boolean < array < object (jbvType's order);
+ *  - an array with more elements sorts higher, an object with more pairs higher; of equal counts, element by element,
+ *    pair by pair in STORAGE order (a key shorter in bytes first, then by its bytes; a key held twice keeps its LAST
+ *    value), each pair's key then its value;
+ *  - a number as a numeric (`10 > 9.5`, `1.0 = 1.00`, `1e2 = 100`, `-0 = 0`), a boolean false < true, a string — a
+ *    key's too — under the DATABASE's default collation (`varstr_cmp(..., DEFAULT_COLLATION_OID)`), whatever a
+ *    column's own collation, ties broken by the bytes;
+ *  - THE RAW SCALAR: a scalar at the top is stored as a one-element array marked `rawScalar`, and the two are compared
+ *    as arrays — the marks first (a raw scalar below an array), then, overriding that, the counts — so `[] < 1 < [1]`
+ *    (the source's comment: "a mild anomaly that an empty top level array sorts less than null"), and every top-level
+ *    scalar sorts below every object.
+ * Equality is that comparison's 0 (`jsonb_eq`), so it is the same key, never the canonical text: `1.0` and `1.00` are
+ * one jsonb written two ways.
+ *
+ * The key is one byte string whose plain byte order (DuckDB's BLOB comparison: bytes, then the shorter first) is that
+ * order, and whose equality is that equality. Every value is prefix-free (it ends where its own encoding says), so a
+ * container is its tag, its count and its members one after another, and the first differing byte is the first token
+ * Postgres finds different:
+ *  - a tag per type, in jbvType's order: null 0x10, string 0x20, number 0x30, boolean 0x40, array 0x50, object 0x60;
+ *  - an array: its tag, its count (4 bytes, big-endian), then 0x00 for a raw scalar and 0x01 for an array (the count
+ *    first: it decides before the mark), then each element; an object: its tag, its count, each pair's key and value;
+ *  - a number: 0x80 for zero; else a sign byte (0x81 positive, 0x7f negative), the decimal exponent E of 0.d₁d₂… × 10^E
+ *    (4 bytes, biased), the digits without leading or trailing zeros, a 0x00 — every byte after the sign inverted for a
+ *    negative number, so a larger magnitude sorts lower;
+ *  - a string: the collation's key of it, then a 0x00 (a jsonb string holds no NUL): its UTF-8 bytes under C (`bytes`);
+ *    under ICU, ICU's sort key (DuckDB's `icu_sort_key`, which the rewrite hands in) and the bytes' hex; under a libc
+ *    locale the measured key (src/duck/libc-order.ts `libcKey`) and the bytes' hex, as the rewrite sorts text
+ *    (src/duck/rewrite.ts `libcKey`).
+ * A string's collation key is computed by DuckDB's ICU for the ICU and libc kinds, which a JavaScript function cannot
+ * call: the rewrite passes the document's strings' ICU keys as a list (`jsonbStrings` lists the strings, in the order
+ * this walk meets them; `icuKeys` the list DuckDB made of them). Held to Postgres by tests/duck-jsonb-order.test.ts
+ * (PGlite's order) and the oracle (tests/integration/duck-oracle.itest.ts, every collation it is run under).
+ */
+import { Buffer } from "node:buffer";
+import { parseJson, storedPairs } from "./jsonb.js";
+import { libcKept, libcKey } from "./libc-order.js";
+const TAG = { null: 0x10, str: 0x20, num: 0x30, bool: 0x40, arr: 0x50, obj: 0x60 };
+const BIAS = 0x80000000;
+/** every string of the value — each object's keys and every string value — in the order the key's walk meets them */
+function eachString(v, out) {
+    switch (v.t) {
+        case "str":
+            out.push(v.v);
+            return;
+        case "arr":
+            for (const x of v.v)
+                eachString(x, out);
+            return;
+        case "obj":
+            for (const [k, x] of storedPairs(v.v)) {
+                out.push(k);
+                eachString(x, out);
+            }
+            return;
+        default: return;
+    }
+}
+/**
+ * THE FUNCTIONS A KEY IS MADE BY, per collation kind (src/duck/functions.ts registers them): the key's, and the one
+ * listing the strings DuckDB makes ICU keys of (none under C). A statement's key (src/duck/rewrite.ts `jsonbKey`) and a
+ * stored one (src/duck/kept.ts `orderKeySql`) both read this table, so the two are one computation by construction;
+ * a stored key calls each name with `STORED_SUFFIX`, the same function answering NULL where this one raises.
+ */
+export const JSONB_KEY_FUNCTIONS = {
+    bytes: { key: "__mason_jsonb_key", strings: null },
+    icu: { key: "__mason_jsonb_key_icu", strings: "__mason_jsonb_strings" },
+    libc: { key: "__mason_jsonb_key_libc", strings: "__mason_jsonb_libc_kept" },
+};
+export const STORED_SUFFIX = "_stored";
+/**
+ * THE KEY'S VERSION: a stored key (src/duck/kept.ts `orderId`) is this encoding's, so a release that changes a byte of
+ * it — a tag, the number's form, a string's — bumps this, and a key stored by an earlier one is dropped at the start
+ * and stored again (tests/duck-jsonb-order.test.ts holds the encoding; a change it catches is a change of this).
+ */
+export const KEY_VERSION = 1;
+/** The strings of a jsonb's text in the key's order (what DuckDB's ICU makes the keys of). */
+export function jsonbStrings(text) {
+    const out = [];
+    eachString(parseJson(text), out);
+    return out;
+}
+/** whether a jsonb's strings hold a character a libc order does not cover: what its key refuses by name
+ *  (`libc-uncovered`, src/duck/libc-order.ts `libcKey`), told from any other failure of the key (src/duck/kept.ts) */
+export function libcUncovered(order, text) {
+    return jsonbStrings(text).some((s) => { for (const ch of s)
+        if (order.uncovered[ch.codePointAt(0)])
+            return true; return false; });
+}
+/** The strings ICU's accent and case levels read for a libc key of each (src/duck/libc-order.ts `libcKept`). */
+export function jsonbLibcKept(order, text) {
+    return jsonbStrings(text).map((s) => libcKept(order, s));
+}
+const u32 = (n) => [(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff];
+/**
+ * A JSON number's text as its key: zero, or its sign, exponent and digits (the value's, exactly: no rounding through a
+ * double, and the scale left out — `1.0` and `1.00` are one number). Read from the text itself, never through a
+ * BigInt of its digits (`1e100000` is one digit and an exponent).
+ */
+export function numberKey(text) {
+    const m = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(text);
+    if (m === null)
+        throw new Error(`not a JSON number: ${JSON.stringify(text)}`);
+    const fraction = m[3] ?? "";
+    // the value is all × 10^(exponent - fraction's length), and so 0.<digits> × 10^E with E = all's length less that
+    const all = (m[2] + fraction).replace(/^0+/, "");
+    const digits = all.replace(/0+$/, "");
+    if (digits === "")
+        return [0x80];
+    const e = all.length - fraction.length + Number(m[4] ?? 0);
+    const body = [...u32(e + BIAS), ...[...digits].map((d) => d.charCodeAt(0)), 0x00];
+    return m[1] === "-" ? [0x7f, ...body.map((b) => 0xff - b)] : [0x81, ...body];
+}
+const hex = (s) => Buffer.from(s, "utf8").toString("hex");
+/** The key of a jsonb's text under the database's collation (the module's header says what it encodes). */
+export function jsonbKey(text, c) {
+    const out = [];
+    let at = 0;
+    const icuKey = () => {
+        if (c.kind === "bytes")
+            throw new Error("a C collation's key needs no ICU key");
+        const k = c.icuKeys[at++];
+        if (k === undefined)
+            throw new Error(`the jsonb holds more strings than the ${c.icuKeys.length} ICU keys handed in: a Mason bug (the rewrite's list and this walk disagree)`);
+        return k;
+    };
+    const str = (s) => {
+        const key = c.kind === "bytes" ? s : c.kind === "icu" ? `${icuKey()} ${hex(s)}` : `${libcKey(c.order, s, icuKey())} ${hex(s)}`;
+        for (const b of Buffer.from(key, "utf8"))
+            out.push(b);
+        out.push(0x00);
+    };
+    const value = (v) => {
+        switch (v.t) {
+            case "null":
+                out.push(TAG.null);
+                return;
+            case "bool":
+                out.push(TAG.bool, v.v ? 1 : 0);
+                return;
+            case "num":
+                out.push(TAG.num, ...numberKey(v.text));
+                return;
+            case "str":
+                out.push(TAG.str);
+                str(v.v);
+                return;
+            case "arr":
+                out.push(TAG.arr, ...u32(v.v.length), 0x01);
+                for (const x of v.v)
+                    value(x);
+                return;
+            case "obj": {
+                const pairs = storedPairs(v.v);
+                out.push(TAG.obj, ...u32(pairs.length));
+                for (const [k, x] of pairs) {
+                    str(k);
+                    value(x);
+                }
+                return;
+            }
+        }
+    };
+    const root = parseJson(text);
+    // a scalar at the top is a raw scalar: a one-element array marked as one (0x00, below an array's 0x01)
+    if (root.t !== "arr" && root.t !== "obj")
+        out.push(TAG.arr, ...u32(1), 0x00);
+    value(root);
+    if (c.kind !== "bytes" && at !== c.icuKeys.length)
+        throw new Error(`the jsonb holds ${at} strings, ${c.icuKeys.length} ICU keys were handed in: a Mason bug (the rewrite's list and this walk disagree)`);
+    return Buffer.from(out);
+}

package/dist/src/duck/jsonb.js

@@ -158,11 +158,19 @@ export function quoteJson(s) {
     }
     return `${out}"`;
 }
-/** jsonb's key order: shorter first, then by the UTF-8 bytes */
+/** jsonb's key order: shorter first, then by the UTF-8 bytes (lengthCompareJsonbStringValue) */
 function keyOrder(a, b) {
     const x = Buffer.from(a, "utf8"), y = Buffer.from(b, "utf8");
     return x.length - y.length || Buffer.compare(x, y);
 }
+/** an object's pairs as jsonb stores them, in its key order: a key held twice keeps its LAST value
+ *  (uniqueifyJsonbObject). What jsonb writes and what its order compares (src/duck/jsonb-order.ts) both read it. */
+export function storedPairs(members) {
+    const last = new Map();
+    for (const [k, x] of members)
+        last.set(k, x);
+    return [...last.keys()].sort(keyOrder).map((k) => [k, last.get(k)]);
+}
 /** a parsed value as jsonb writes it */
 export function writeJson(v) {
     switch (v.t) {
@@ -171,13 +179,7 @@ export function writeJson(v) {
         case "num": return numericText(parseNumeric(v.text));
         case "str": return quoteJson(v.v);
         case "arr": return `[${v.v.map(writeJson).join(", ")}]`;
-        case "obj": {
-            // a duplicate key keeps its LAST value (jsonb's uniqueifyJsonbObject)
-            const last = new Map();
-            for (const [k, x] of v.v)
-                last.set(k, x);
-            return `{${[...last.keys()].sort(keyOrder).map((k) => `${quoteJson(k)}: ${writeJson(last.get(k))}`).join(", ")}}`;
-        }
+        case "obj": return `{${storedPairs(v.v).map(([k, x]) => `${quoteJson(k)}: ${writeJson(x)}`).join(", ")}}`;
     }
 }
 /** A JSON text as Postgres's jsonb writes it; throws a json syntax error for a text that is no JSON. */

package/dist/src/duck/kept-route.js

@@ -32,7 +32,7 @@ export function canonicalRead(r) {
 }
 /** a read as one string: the request log's key (its `->>` or `->` included: the policy counts both for one path) */
 export const readKey = (r) => { const c = canonicalRead(r); return JSON.stringify([c.source, c.column, c.steps, c.either, c.canonical, c.asText]); };
-/** a read's key, back (null for a string that is not one) */
+/** a read's key, back (null for a string that is not one, and for a read of a kind this Mason does not know: `knownSteps`) */
 export function readOfKey(key) {
     let v;
     try {
@@ -46,8 +46,27 @@ export function readOfKey(key) {
     const [source, column, steps, either, canonical, asText] = v;
     if (typeof source !== "string" || typeof column !== "string" || typeof steps !== "string" || typeof either !== "boolean" || typeof canonical !== "boolean" || typeof asText !== "boolean")
         return null;
+    if (!knownSteps(steps))
+        return null;
     return canonicalRead({ source, column, steps, either, canonical, asText });
 }
+/**
+ * WHETHER A READ'S STEPS ARE A KIND THIS MASON KNOWS: a path's (a JSON array of keys and indexes) or a jsonb's key
+ * (`isKeyRead`). A request log or a ledger written by another release may hold a kind this one does not know (an
+ * earlier release met `equal:…` and `order:…` and failed every write of the table, and its status, parsing them as a
+ * path): such a read is INERT, never fatal — not counted (`readOfKey`), its ledger row dropped at the start with its
+ * column (src/duck/kept.ts `open`), and said raw (`pathText`).
+ */
+export function knownSteps(steps) {
+    if (isKeyRead({ steps }))
+        return true;
+    try {
+        return Array.isArray(JSON.parse(steps));
+    }
+    catch {
+        return false;
+    } // not JSON: not a path, so not a kind known here
+}
 /**
  * THE STORED PATH A READ IS ANSWERED FROM: its canonical read (`canonicalRead`) without `->>`. One column serves a
  * path's `->` and its `->>` (the test server: a popular key was stored twice, as `-> 'k'` and `->> 'k'`, two
@@ -72,9 +91,48 @@ export const hiddenName = (r) => `${HIDDEN_PREFIX}${createHash("sha256").update(
  * table name, so `public.orders.id` and `orders.id` still bind.
  */
 export const VISIBLE_CATALOG = "__mason_visible";
+/**
+ * A JSONB'S ORDER, AS A READ (src/duck/jsonb-order.ts; docs/DESIGN.md "jsonb's order"): `ORDER BY j`, `GROUP BY j`,
+ * `j = k` compare a key of the whole document, computed per row by a function — 19 s for `ORDER BY j LIMIT 100` over
+ * 20 M rows under C, 83 s under a libc locale, where the same sort from a stored key takes 0.1 s. So the key is a read
+ * like a path's, recorded, weighed and stored by the same keeper: its `steps` is `ORDER_STEPS` (never a path's, which
+ * is a JSON array). The rewrite records it as `ORDER_STEPS`; the keeper stores it as `ORDER_STEPS:<the collation's
+ * id>` (src/duck/kept.ts `own`), because a string's key is its collation's, so a key stored under another collation is
+ * another column, never read.
+ */
+export const ORDER_STEPS = "order";
+/** a stored key's marks (hex, DuckDB's `unhex`): a NULL document's, and a row whose key raised, computed when read. No
+ *  key begins with a 0x00 (src/duck/jsonb-order.ts: every key starts with an array's or an object's tag) */
+export const KEY_NULL_HEX = "00";
+export const KEY_RECOMPUTE_HEX = "0021";
+export const isOrderRead = (r) => r.steps === ORDER_STEPS || r.steps.startsWith(`${ORDER_STEPS}:`);
+/**
+ * A JSONB'S EQUALITY, AS A READ, under a LIBC collation alone (src/duck/rewrite.ts `equalKey`): `GROUP BY j`, `DISTINCT
+ * j`, `j = k`, `j IN (…)`, a join on `=`, `count(DISTINCT j)` compare the C key — a libc order's key refuses a string
+ * its measured order does not cover, where equality never needed an order (docs/DESIGN.md "jsonb's order"), so the
+ * order's stored key cannot serve them, and over 20 M rows a GROUP BY computing the C key per row took 32–45 s of CPU
+ * (34–77 s wall; docs/DESIGN.md "Under libc, the C key stored for equality"), 0.15–0.23 s from it. So the C key
+ * is a read of its own, stored beside the order's as `EQUAL_STEPS:<id>` (src/duck/kept.ts `equalId`). Under C and ICU
+ * the order's key equates exactly and never refuses, so equality reads IT and this read is never recorded nor stored
+ * (src/duck/kept.ts `own` folds one into the order's).
+ */
+export const EQUAL_STEPS = "equal";
+export const isEqualRead = (r) => r.steps === EQUAL_STEPS || r.steps.startsWith(`${EQUAL_STEPS}:`);
+/** a jsonb's key, the order's or the equality's: a BLOB column of the key's bytes, its marks `KEY_NULL_HEX` and
+ *  `KEY_RECOMPUTE_HEX`, stored under an id of what its bytes depend on */
+export const isKeyRead = (r) => isOrderRead(r) || isEqualRead(r);
 /** a read as a developer writes it: `payload ->> 'customer_id'`, `payload #> '{a,0}'`; a stored path (no `asText`) as
- *  its `->` read */
+ *  its `->` read; a jsonb's order and equality as what reads each (under libc the order's key sorts and the C key
+ *  equates, two columns: `mason status` names each by its own) */
 export function pathText(r) {
+    // a kind this Mason does not know (another release's): said as it is stored, never parsed (`knownSteps`)
+    if (!knownSteps(r.steps))
+        return `${r.column} (stored as ${r.steps}: a kind this Mason does not know)`;
+    if (isEqualRead(r))
+        return `GROUP BY ${r.column} (its jsonb equality: grouped, DISTINCT, = and IN)`;
+    if (isOrderRead(r))
+        return r.steps.startsWith(`${ORDER_STEPS}:libc:`) ? `ORDER BY ${r.column} (its jsonb order: sorted, and compared by < and >)`
+            : `ORDER BY ${r.column} (its jsonb order: sorted, grouped and compared)`;
     const steps = JSON.parse(r.steps);
     const op = (one) => one ? (r.asText ? "->>" : "->") : (r.asText ? "#>>" : "#>");
     if (steps.length === 1 && !r.either) {
@@ -84,6 +142,28 @@ export function pathText(r) {
     const element = (s) => typeof s === "number" || /^[A-Za-z0-9_]+$/.test(s) ? String(s) : `"${String(s).replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"`;
     return `${r.column} ${op(false)} '{${steps.map(element).join(",").replaceAll("'", "''")}}'`;
 }
+/**
+ * WHAT AN EXACT READ RAISES AT A MARK (src/duck/rewrite.ts `keptRead`, `keptKey`): a row marked to be computed
+ * (`KEPT_RECOMPUTE`, `KEY_RECOMPUTE_HEX`) written AFTER the path was counted exact — by the stream, the poll or a copy's
+ * side table, before the detector (src/duck/kept.ts `check`) counts it and turns the reads to the CASE — is never read
+ * as a value: the exact read raises at it, by the column alone. A jsonb's key under a libc collation is marked where
+ * its strings hold a character the measured order does not cover, the gap the statement's own key refuses by name
+ * (`libc-uncovered`); any other mark is a Mason bug. Once counted (a minute at most), the CASE computes the row from its
+ * document, and raises the function's own error (the character named) or answers. Raised by `MARKED_FUNCTIONS`, never
+ * DuckDB's `error()`, which is volatile (a window's ORDER BY refuses it: the eager oracle's `rank() OVER (ORDER BY j)`).
+ */
+export function markedError(r, libc) {
+    if (isOrderRead(r) && libc) {
+        return { code: "0A000", message: "a jsonb compared or sorted here holds a string Mason cannot order as the database's collation does (a combining mark, a Thai or " +
+                "Lao vowel written before its consonant, a private-use or unassigned code point), in a row written since its stored keys were counted; asked again " +
+                "in a minute, the error names the character (Mason gap libc-uncovered)" };
+    }
+    return { code: "XX000", message: `a stored value of ${pathText(r)} could not be computed when its row was written (a Mason bug: report it, with mason logs); ` +
+            "asked again in a minute, it is computed from the row's document" };
+}
+/** the functions that raise `markedError` at a marked row (src/duck/functions.ts): of the column read (a key as its hex,
+ *  so neither is a constant DuckDB would fold), the code and the message; a key's answers a BLOB, a path's a VARCHAR */
+export const MARKED_FUNCTIONS = { key: "__mason_marked_key", value: "__mason_marked_value" };
 const isObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
 const kindOf = (n) => isObject(n) && Object.keys(n).length === 1 && /^[A-Z]/.test(Object.keys(n)[0]) ? Object.keys(n)[0] : null;
 const bodyOf = (n) => n[kindOf(n)];