@mason-data/cli 0.7.0 → 0.9.0

package/dist/src/duck/grants.js

@@ -0,0 +1,176 @@
+/**
+ * ROLES: WHAT A USER MAY READ, HELD ON THE ANALYSED TREE (GitHub issue 30, stage 1: table and column grants per role).
+ * A role is an entry of the project's roles file (`mason/roles.yml`, src/cli/roles.ts): for each copied table, every
+ * column (`{}`), `columns:` or `exclude:`, the sources file's own words — or `tables: all`, every copied table and column.
+ * A `mason user` carries a role (src/duck/users.ts `MasonUser.role`); a user with none is held to the file's
+ * `default:` role, and reads nothing when the file names none (stage 1b: the FILE decides, never the code); with no
+ * roles file at all, every user reads every copied table. A token (no user) reads every copied table.
+ *
+ * ENFORCED ON THE TREE POSTGRES ANALYSED, NEVER ON SQL TEXT: Postgres puts one `RTEPERMISSIONINFO` per relation in the
+ * query tree it analyses (src/duck/analyze.ts), with `selectedCols` as its own bitmap of the columns the statement
+ * reads — a column read inside a CTE, a subquery, a join's USING, a window, a WHERE or a CASE is in it; `SELECT *` lists
+ * every column; `count(*)` reads none; a whole-row reference (`SELECT t FROM t`) is the bit for attnum 0, which means
+ * every column. This is the information Postgres's own executor checks privileges with (ExecCheckPermissions), so an
+ * alias, a subquery or a view cannot get around it: a view is expanded by Mason itself before the check
+ * (src/duck/views.ts), so the check sees the tables it reads — stricter than Postgres, where a grant on the view
+ * suffices, and said so in docs/POSTGRES-API.md.
+ *
+ * THE REFUSAL IS POSTGRES'S: 42501 `permission denied for table orders`, the words Postgres 18 answers whether the
+ * table or one of its columns is missing from the grants (its executor names the table only), so a tool handles it as
+ * it handles Postgres's. The DETAIL names the role and the first hidden column, the HINT what the role may read. The
+ * rules are Postgres's too: `count(*)` over a table is answered when the role has any column of it; `SELECT *` over a
+ * table the role has only part of is refused, never narrowed (a narrowed `*` would change the result's columns); a
+ * system column (xmin, ctid) needs the whole table.
+ */
+import { StatementError } from "./statement.js";
+import { catalogRows } from "./analyze.js";
+/** no roles file: every session reads every copied table, as before; one object, so the listener sees no change */
+export const NO_ROLES = { file: null, grants: new Map(), problem: null, default: null };
+/**
+ * THE CATALOG'S ROLE OF A USER THE FILE HOLDS TO NOTHING (no role of its own, no `default:`): a Postgres role with no
+ * grant, so its catalog requests answer as Postgres answers a login with none — information_schema lists none of the
+ * copied tables or views, every `has_*_privilege` is false, `set_config` is revoked (while any role exists; a session's
+ * own `set_config` never reaches the catalog, src/api/postgres/translate.ts setConfig).
+ * Named outside the file's namespace: `ROLE_NAME_RE` (src/cli/roles.ts) forbids a leading underscore, so no role of the
+ * file can ever be it.
+ */
+export const NO_ROLE_CATALOG = "__mason_no_role";
+/** The roles the catalog Postgres holds (src/api/postgres/catalog.ts catalogRoles): the file's, and — whenever a roles
+ *  file exists — `NO_ROLE_CATALOG`, a role with no table, for a user the file holds to nothing. */
+export function catalogRolesOf(live) {
+    if (live.file === null)
+        return live.grants;
+    return new Map([...live.grants, [NO_ROLE_CATALOG, { name: NO_ROLE_CATALOG, file: live.file, tables: new Map(), all: false }]]);
+}
+/** `FirstLowInvalidHeapAttributeNumber` is -7: a column's bit in `selectedCols` is its attnum + 7 */
+const ATTNUM_OFFSET = 7;
+/** ACL_SELECT in `requiredPerms` */
+const ACL_SELECT = 2;
+const isNode = (v) => v !== null && v !== undefined && typeof v === "object" && !Array.isArray(v);
+/** Every RTEPERMISSIONINFO of the tree that asks SELECT, with its bitmap read as attnums. */
+export function tableReads(tree) {
+    const out = [];
+    const visit = (v) => {
+        if (Array.isArray(v)) {
+            for (const x of v)
+                visit(x);
+            return;
+        }
+        if (!isNode(v))
+            return;
+        if (v._node === "RTEPERMISSIONINFO") {
+            const perms = Number(v.requiredPerms ?? 0);
+            const relid = Number(v.relid ?? 0);
+            if ((perms & ACL_SELECT) !== 0 && relid > 0) {
+                // a bitmapset prints as `(b 8 9)`: the `b` first, then each member
+                const bits = Array.isArray(v.selectedCols) ? v.selectedCols.filter((x) => typeof x === "string" && /^\d+$/.test(x)).map(Number) : [];
+                const read = { relid, attnums: [], wholeRow: false, system: false };
+                for (const b of bits) {
+                    const attnum = b - ATTNUM_OFFSET;
+                    if (attnum === 0)
+                        read.wholeRow = true;
+                    else if (attnum < 0)
+                        read.system = true;
+                    else
+                        read.attnums.push(attnum);
+                }
+                out.push(read);
+            }
+        }
+        for (const [k, x] of Object.entries(v))
+            if (k !== "_node")
+                visit(x);
+    };
+    visit(tree);
+    return out;
+}
+/**
+ * The reads named through the catalog Postgres's own pg_class and pg_attribute (dropped columns left out, as Postgres
+ * leaves them): the copied tables only (relkind r) — a view is never here, since Mason expands it before the plan is
+ * made (src/duck/views.ts), and a catalog relation is not a copy.
+ */
+export async function resolveReads(catalog, reads) {
+    if (reads.length === 0)
+        return [];
+    const relids = [...new Set(reads.map((r) => Math.trunc(r.relid)))];
+    const rows = await catalogRows(catalog, `SELECT c.oid::text, n.nspname, c.relname, a.attnum::text, a.attname FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace ` +
+        `LEFT JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped WHERE c.oid IN (${relids.join(", ")}) AND c.relkind = 'r' ` +
+        // the copied tables alone: pg_class and the rest of pg_catalog are relkind 'r' too, and PUBLIC's on Postgres (a review finding of #35)
+        `AND n.nspname NOT IN ('pg_catalog', 'information_schema') ORDER BY c.oid, a.attnum`);
+    const tables = new Map();
+    for (const [oid, schema, name, attnum, attname] of rows) {
+        const t = tables.get(Number(oid)) ?? { table: `${schema}.${name}`, columns: new Map() };
+        if (attnum !== null && attnum !== undefined && attname !== null && attname !== undefined)
+            t.columns.set(Number(attnum), attname);
+        tables.set(Number(oid), t);
+    }
+    const out = [];
+    for (const r of reads) {
+        const t = tables.get(r.relid);
+        if (t === undefined)
+            continue;
+        const all = [...t.columns.entries()].sort((a, b) => a[0] - b[0]).map(([, n]) => n);
+        const named = r.wholeRow ? all : [...r.attnums].sort((a, b) => a - b).map((n) => t.columns.get(n)).filter((n) => n !== undefined);
+        out.push({ table: t.table, columns: named, all, system: r.system });
+    }
+    return out;
+}
+/** a table as a tool writes it: its name alone in public, else schema.table (the sources file's own spelling) */
+const shown = (table) => table.startsWith("public.") ? table.slice("public.".length) : table;
+/** The 42501 of a read the role does not cover, Postgres's words first; null when the role covers every read. */
+export function grantRefusal(reads, role) {
+    if (role.noRole !== undefined)
+        return reads.length === 0 ? null : noRoleRefusal(reads[0].table, role.file, role.noRole);
+    const tablesOf = () => {
+        if (role.all)
+            return "every synced table (tables: all)";
+        const names = [...role.tables.keys()].map(shown);
+        return names.length ? names.join(", ") : "none";
+    };
+    for (const r of reads) {
+        const g = role.tables.get(r.table);
+        const denied = (detail, hint) => new StatementError({ code: "42501", message: `permission denied for table ${r.table.replace(/^[^.]+\./, "")}`, detail, hint });
+        if (g === undefined) {
+            return denied(`the role ${role.name} has no grant on ${shown(r.table)}`, `the role ${role.name} reads: ${tablesOf()} (${role.file} names what each role reads; Mason gap roles)`);
+        }
+        if (g.all)
+            continue;
+        const may = `the role ${role.name} reads these columns of ${shown(r.table)}: ${g.columns.join(", ")} (${role.file} names what each role reads; Mason gap roles)`;
+        if (r.system)
+            return denied(`the role ${role.name} may read columns of ${shown(r.table)}, and a system column needs the whole table`, may);
+        const hidden = r.columns.find((c) => !g.columns.includes(c));
+        if (hidden !== undefined)
+            return denied(`the role ${role.name} may not read column ${hidden} of ${shown(r.table)}`, may);
+        // no column read (count(*)): any column of the table is enough, as Postgres rules it, and a grant names at least one
+    }
+    return null;
+}
+/**
+ * THE 42501 OF A USER WITH NO ROLE UNDER A FILE THAT NAMES NO DEFAULT (stage 1b): Postgres's message for the first
+ * table read, the reason in the DETAIL, and both ways out as the exact lines in the HINT — the file's `default:`, or
+ * the user made again with a role. `file`: the roles file as messages name it.
+ */
+export function noRoleRefusal(table, file, who) {
+    return new StatementError({ code: "42501", message: `permission denied for table ${table.replace(/^[^.]+\./, "")}`,
+        detail: `the user "${who.user}" has no role and ${file} names no default`, hint: noRoleWayOut(who.user, file, who.example) });
+}
+/** The two ways out for a user with no role under a file with no default, as exact lines: ONE wording, for the
+ *  refusal's HINT, `mason start`, `mason user list` and `mason user add`. */
+export const noRoleWayOut = (user, file, example) => `write default: ${example} at the top of ${file}, or mason user remove ${user} and mason user add ${user} --role ${example}`;
+/** A role of the file to name in a way out: one of `tables: all` where there is one (the role today's access is), else
+ *  the first, or a placeholder where it names none. */
+export const exampleRole = (live) => [...live.grants.values()].find((g) => g.all)?.name ?? [...live.grants.keys()][0] ?? "<a role>";
+/**
+ * The statement's table reads held to the role: the tree's reads named through the catalog, then refused where the role
+ * does not cover one. Called by the plan (src/duck/plan.ts) after Postgres's analysis and the views' expansion, before
+ * the rewrite — so Postgres's own refusal (an unknown column) comes first, and a role's before a Mason gap.
+ */
+export async function enforceGrants(catalog, tree, role) {
+    const reads = await resolveReads(catalog, tableReads(tree));
+    if (role !== null) {
+        const refused = grantRefusal(reads, role);
+        if (refused !== null)
+            throw refused;
+    }
+    return reads;
+}

package/dist/src/duck/held-types.js

@@ -0,0 +1,64 @@
+/**
+ * WHAT THE COPY HOLDS FOR A POSTGRES TYPE (issues 126 and 129): the one list that says so, read by the copy
+ * (src/duck/sync.ts: what it reads and declares), the catalog (the column declared as the source spells it, always), the
+ * wire (src/duck/types.ts: the source's OID) and the rewrite (src/duck/rewrite.ts: a literal, a sort).
+ *
+ * THE CLASS: a type the copy holds as another type. The catalog said the wire's type, and the wire said what DuckDB
+ * happened to read (`timetz` as text over DuckDB's TIMETZ, `point` as text over a STRUCT, `box` not copied at all), so a
+ * tool believing the catalog sent text functions, and they failed in the engine's words.
+ *
+ * THE DECISION: the catalog reports the source's type, whatever the copy holds, so Postgres's own analysis decides every
+ * statement; and every type is one of two kinds:
+ *  - `same` (`NATIVE`): DuckDB holds a value of the type itself, whose order and text are Postgres's;
+ *  - `text`: DuckDB holds Postgres's text of the value (`c::text`, an array as `c::text[]`). That is anything `NATIVE`
+ *    does not name, a type added to Postgres or made by a user included: nothing falls through to what DuckDB reads.
+ * What can be answered over a text is answered (the value, `::text`, `=` and GROUP BY where equal values have one text); an
+ * order is not the text's, so a sort is refused by name (`held-type-compare`), and every operator, aggregate and cast
+ * stays refused until the oracle has verified it (src/duck/verified.ts).
+ *
+ * `tests/duck-held-types.test.ts` holds this list to pg_type: a type of Postgres's named neither way fails it.
+ */
+/** the types DuckDB holds as themselves, as `format_type` spells them with their typmod left off */
+const NATIVE = new Set(["smallint", "integer", "bigint", "real", "double precision", "numeric", "text", "character varying", "character", "name", '"char"',
+    "boolean", "uuid", "date", "time without time zone", "time with time zone", "timestamp without time zone", "timestamp with time zone", "interval",
+    "json", "jsonb", "bytea", "oid",
+    // held as their text from the first: an inet's output is the rewrite's and src/duck/types.ts's, a citext's its bytes
+    "inet", "citext"]);
+const held = (oid, array, size, textEqual = true, literal = true) => ({ oid, array, size, textEqual, literal });
+/** every built-in type a column can have that `NATIVE` does not name, by its pg_type.typname */
+export const HELD = {
+    point: held(600, 1017, 16, false), lseg: held(601, 1018, 32, false), path: held(602, 1019, -1, false), box: held(603, 1020, 32, false), polygon: held(604, 1027, -1, false),
+    line: held(628, 629, 24, false), circle: held(718, 719, 24, false), money: held(790, 791, 8, true, false), macaddr: held(829, 1040, 6), macaddr8: held(774, 775, 8), cidr: held(650, 651, -1),
+    bit: held(1560, 1561, -1), varbit: held(1562, 1563, -1), tsvector: held(3614, 3643, -1), tsquery: held(3615, 3645, -1, false), xml: held(142, 143, -1, false),
+    jsonpath: held(4072, 4073, -1, false), tid: held(27, 1010, 6), xid: held(28, 1011, 4), cid: held(29, 1012, 4), xid8: held(5069, 271, 8), pg_lsn: held(3220, 3221, 8),
+    regclass: held(2205, 2210, 4), regtype: held(2206, 2211, 4), regproc: held(24, 1008, 4), regprocedure: held(2202, 2207, 4), regoper: held(2203, 2208, 4), regoperator: held(2204, 2209, 4),
+    regconfig: held(3734, 3735, 4), regdictionary: held(3769, 3770, 4), regnamespace: held(4089, 4090, 4), regrole: held(4096, 4097, 4), regcollation: held(4191, 4192, 4),
+    int4range: held(3904, 3905, -1), int8range: held(3926, 3927, -1), daterange: held(3912, 3913, -1), tsrange: held(3908, 3909, -1), tstzrange: held(3910, 3911, -1, false),
+    numrange: held(3906, 3907, -1, false), int4multirange: held(4451, 6150, -1), int8multirange: held(4536, 6157, -1), datemultirange: held(4535, 6155, -1), tsmultirange: held(4533, 6152, -1),
+    tstzmultirange: held(4534, 6153, -1, false), nummultirange: held(4532, 6151, -1, false), int2vector: held(22, 1006, -1), oidvector: held(30, 1013, -1), aclitem: held(1033, 1034, 16),
+    refcursor: held(1790, 2201, -1), pg_snapshot: held(5038, 5039, -1, false), txid_snapshot: held(2970, 2949, -1, false),
+};
+/** `oid`'s array: DuckDB reads no array of it, so the copy holds it as a list of its digits */
+export const OID_ARRAY = 1028;
+const HELD_OIDS = new Map(Object.values(HELD).map((h) => [h.oid, h]));
+const HELD_ARRAYS = new Map([...Object.values(HELD).map((h) => [h.array, h]), [OID_ARRAY, held(26, OID_ARRAY, -1)]]);
+/** a built-in type held as its text, by its OID or its array's */
+export const heldType = (oid) => oid === null ? undefined : HELD_OIDS.get(oid) ?? HELD_ARRAYS.get(oid);
+/** a held type's name, for a refusal */
+export const heldName = (oid) => Object.entries(HELD).find(([, h]) => h.oid === oid || h.array === oid)?.[0] ?? (oid === 18 ? '"char"' : oid === OID_ARRAY ? "oid[]" : String(oid));
+/** an array of a type held as its text: a list of each element's text */
+export const isHeldArray = (oid) => oid !== null && HELD_ARRAYS.has(oid);
+/** a time written in the session's zone, which the copy's text cannot be: written again where the statement executes */
+export const ZONED_HELD = new Map([[3910, "tstzrange"], [4534, "tstzmultirange"]]);
+/** what `format_type` writes, its typmod and `[]` left off (`time(3) with time zone`, `bit(8)`, `numeric(12,2)[]`) */
+const baseOf = (spelled) => spelled.replace(/\[\]$/, "").replace(/\(\d+(,\d+)?\)/, "").replace(/\s+/g, " ").trim().replace(/^interval .*/, "interval");
+/** whether DuckDB holds a value of this type as itself */
+export const isNative = (spelled) => NATIVE.has(baseOf(spelled));
+/** How the copy holds a column of the source's type `spelled` (`typeKind`: pg_type.typtype of the type or its element).
+ *  An enum is DuckDB's ENUM; any other type a database made is its text. */
+export function holdOf(spelled, typeKind) {
+    const array = spelled.endsWith("[]");
+    if (typeKind === "e" || (isNative(spelled) && !(array && baseOf(spelled) === "oid")))
+        return "same";
+    return array ? "text[]" : "text";
+}

package/dist/src/duck/history.js

@@ -0,0 +1,127 @@
+import { MASON_SCHEMA } from "./store.js";
+/** How many days every history table keeps (the request log, the sync's events, the usage samples, the copies'
+ *  sizes) unless MASON_HISTORY_DAYS says. */
+export const DEFAULT_HISTORY_DAYS = 7;
+/** The instant `days` days before `now` (ms since the epoch), ISO 8601 in UTC: what a history table's trim deletes
+ *  before. `now` is the module's own clock, the one that stamps its rows, never the engine's `now()`: a module reads
+ *  ONE clock, or a clock it is given (a test's, a run's) and the engine's disagree and the trim deletes what it keeps. */
+export function cutoff(now, days) { return new Date(now - days * 86_400_000).toISOString(); }
+const KEPT = "the request log, the sync's events, the usage samples and the synced tables' sizes";
+/** The days MASON_HISTORY_DAYS keeps, or what is wrong with the text. Unset or empty: the default. */
+export function historyDays(text) {
+    if (text === undefined || text.trim() === "")
+        return DEFAULT_HISTORY_DAYS;
+    const days = Number(text);
+    if (!Number.isInteger(days) || days < 1) {
+        return { message: `MASON_HISTORY_DAYS=${JSON.stringify(text)} is not a number of days`,
+            why: `it is how many days Mason keeps its history (${KEPT}): a whole number, 1 or more`,
+            fix: `write MASON_HISTORY_DAYS=${DEFAULT_HISTORY_DAYS} (the default, as when it is unset) or another whole number of days` };
+    }
+    return days;
+}
+/**
+ * THE HISTORY'S DAYS, AS THE ENVIRONMENT SETS THEM — read by this one function at every entrance (`mason serve`'s
+ * settings report, `mason start` before it spawns, and the run before anything opens), so a wrong setting is refused
+ * before the copy is touched, in the Failure shape. `MASON_REQUEST_LOG_DAYS` SET IS REFUSED, never run on the default
+ * (CLAUDE.md: a renamed setting refuses the start, naming the line to write): it kept the request log alone once, and
+ * the days kept are every history table's now, so it is MASON_HISTORY_DAYS. Then MASON_HISTORY_DAYS must be a number
+ * of days (`historyDays`).
+ */
+export function historyDaysSetting(env) {
+    const v = env.MASON_REQUEST_LOG_DAYS;
+    if (v !== undefined && v !== "") {
+        return { message: `MASON_REQUEST_LOG_DAYS=${v} is no longer read`,
+            why: `it was renamed MASON_HISTORY_DAYS, since the days it keeps are every history's (${KEPT})`,
+            fix: `write MASON_HISTORY_DAYS=${v.trim()} instead, then unset MASON_REQUEST_LOG_DAYS` };
+    }
+    return historyDays(env.MASON_HISTORY_DAYS);
+}
+/** what is wrong, as one line of `mason serve`'s settings report */
+export function historyDaysWrongText(w) {
+    return `${w.message}: ${w.why} — ${w.fix}`;
+}
+/** how often the rows past the retention are deleted (and once at open) */
+export const TRIM_EVERY_MS = 60 * 60 * 1000;
+/** A history table's connections and its hourly trim. */
+export class History {
+    writer;
+    reader;
+    trimTimer = null;
+    constructor(writer, reader) { this.writer = writer; this.reader = reader; }
+    /** The writer connected, `_mason` and each of `ddl` run on it in order, then the reader connected. */
+    static async open(store, ddl) {
+        const writer = await store.instance.connect();
+        await writer.run(`CREATE SCHEMA IF NOT EXISTS ${MASON_SCHEMA}`);
+        for (const d of ddl)
+            await writer.run(d);
+        const reader = await store.instance.connect();
+        return new History(writer, reader);
+    }
+    /** `trim` now (its failure is the open's), then every TRIM_EVERY_MS, an hourly one that fails said by `failed`. */
+    async keepTrimmed(trim, failed) {
+        await trim();
+        this.trimTimer = setInterval(() => { trim().catch((e) => failed(e)); }, TRIM_EVERY_MS);
+        this.trimTimer.unref();
+    }
+    /** No more hourly trims (the first step of a close). */
+    stopTrimming() {
+        if (this.trimTimer !== null) {
+            clearInterval(this.trimTimer);
+            this.trimTimer = null;
+        }
+    }
+    /** Both connections closed (the last step of a close). */
+    closeConnections() {
+        this.writer.closeSync();
+        this.reader.closeSync();
+    }
+}
+/** A streak of failures, said once: `failed` says the first of a streak and counts what each cost; `succeeded` says,
+ *  once, what the streak cost, and ends it. */
+export class Streak {
+    lost = 0;
+    failed(cost, sayFirst) {
+        if (this.lost === 0)
+            sayFirst();
+        this.lost += cost;
+    }
+    succeeded(sayEnd) {
+        if (this.lost > 0) {
+            sayEnd(this.lost);
+            this.lost = 0;
+        }
+    }
+}
+/** Rows written in batches: `push` one; a batch is written by `write` once `rows` are waiting or after `ms`, one batch
+ *  at a time. `write` must not throw: a failure is its own to say (a Streak). */
+export class Batches {
+    buffer = [];
+    timer = null;
+    /** the write in flight: one at a time, so batches commit in the order they were taken */
+    flushing = Promise.resolve();
+    rows;
+    ms;
+    write;
+    constructor(o) { this.rows = o.rows; this.ms = o.ms; this.write = o.write; }
+    push(row) {
+        this.buffer.push(row);
+        if (this.buffer.length >= this.rows) {
+            void this.flush();
+            return;
+        }
+        this.timer ??= setTimeout(() => { void this.flush(); }, this.ms);
+    }
+    /** Writes what is waiting, after any write in flight. Resolves whether it was written or not (`write` says a failure). */
+    flush() {
+        if (this.timer !== null) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+        const rows = this.buffer;
+        this.buffer = [];
+        if (rows.length === 0)
+            return this.flushing;
+        this.flushing = this.flushing.then(() => this.write(rows));
+        return this.flushing;
+    }
+}

package/dist/src/duck/jsonb-order.js

@@ -30,14 +30,14 @@
  *    under ICU, ICU's sort key (DuckDB's `icu_sort_key`, which the rewrite hands in) and the bytes' hex; under a libc
  *    locale the measured key (src/duck/libc-order.ts `libcKey`) and the bytes' hex, as the rewrite sorts text
  *    (src/duck/rewrite.ts `libcKey`).
- * A string's collation key is computed by DuckDB's ICU for the ICU and libc kinds, which a JavaScript function cannot
+ * A string's collation key is computed by DuckDB's ICU for the ICU kind, which a JavaScript function cannot
  * call: the rewrite passes the document's strings' ICU keys as a list (`jsonbStrings` lists the strings, in the order
  * this walk meets them; `icuKeys` the list DuckDB made of them). Held to Postgres by tests/duck-jsonb-order.test.ts
  * (PGlite's order) and the oracle (tests/integration/duck-oracle.itest.ts, every collation it is run under).
  */
 import { Buffer } from "node:buffer";
 import { parseJson, storedPairs } from "./jsonb.js";
-import { libcKept, libcKey } from "./libc-order.js";
+import { libcKey } from "./libc-order.js";
 const TAG = { null: 0x10, str: 0x20, num: 0x30, bool: 0x40, arr: 0x50, obj: 0x60 };
 const BIAS = 0x80000000;
 /** every string of the value — each object's keys and every string value — in the order the key's walk meets them */
@@ -68,7 +68,7 @@ function eachString(v, out) {
 export const JSONB_KEY_FUNCTIONS = {
     bytes: { key: "__mason_jsonb_key", strings: null },
     icu: { key: "__mason_jsonb_key_icu", strings: "__mason_jsonb_strings" },
-    libc: { key: "__mason_jsonb_key_libc", strings: "__mason_jsonb_libc_kept" },
+    libc: { key: "__mason_jsonb_key_libc" },
 };
 export const STORED_SUFFIX = "_stored";
 /**
@@ -86,14 +86,11 @@ export function jsonbStrings(text) {
 /** whether a jsonb's strings hold a character a libc order does not cover: what its key refuses by name
  *  (`libc-uncovered`, src/duck/libc-order.ts `libcKey`), told from any other failure of the key (src/duck/kept.ts) */
 export function libcUncovered(order, text) {
-    return jsonbStrings(text).some((s) => { for (const ch of s)
+    return jsonbStrings(text).some((s) => { if (order.unread)
+        return true; for (const ch of s)
         if (order.uncovered[ch.codePointAt(0)])
             return true; return false; });
 }
-/** The strings ICU's accent and case levels read for a libc key of each (src/duck/libc-order.ts `libcKept`). */
-export function jsonbLibcKept(order, text) {
-    return jsonbStrings(text).map((s) => libcKept(order, s));
-}
 const u32 = (n) => [(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff];
 /**
  * A JSON number's text as its key: zero, or its sign, exponent and digits (the value's, exactly: no rounding through a
@@ -120,15 +117,15 @@ export function jsonbKey(text, c) {
     const out = [];
     let at = 0;
     const icuKey = () => {
-        if (c.kind === "bytes")
-            throw new Error("a C collation's key needs no ICU key");
+        if (c.kind !== "icu")
+            throw new Error("only ICU's key is handed in");
         const k = c.icuKeys[at++];
         if (k === undefined)
             throw new Error(`the jsonb holds more strings than the ${c.icuKeys.length} ICU keys handed in: a Mason bug (the rewrite's list and this walk disagree)`);
         return k;
     };
     const str = (s) => {
-        const key = c.kind === "bytes" ? s : c.kind === "icu" ? `${icuKey()} ${hex(s)}` : `${libcKey(c.order, s, icuKey())} ${hex(s)}`;
+        const key = c.kind === "bytes" ? s : c.kind === "icu" ? `${icuKey()} ${hex(s)}` : `${libcKey(c.order, s)} ${hex(s)}`;
         for (const b of Buffer.from(key, "utf8"))
             out.push(b);
         out.push(0x00);
@@ -169,7 +166,7 @@ export function jsonbKey(text, c) {
     if (root.t !== "arr" && root.t !== "obj")
         out.push(TAG.arr, ...u32(1), 0x00);
     value(root);
-    if (c.kind !== "bytes" && at !== c.icuKeys.length)
+    if (c.kind === "icu" && at !== c.icuKeys.length)
         throw new Error(`the jsonb holds ${at} strings, ${c.icuKeys.length} ICU keys were handed in: a Mason bug (the rewrite's list and this walk disagree)`);
     return Buffer.from(out);
 }

package/dist/src/duck/kept-route.js

@@ -149,14 +149,14 @@ export function pathText(r) {
  * as a value: the exact read raises at it, by the column alone. A jsonb's key under a libc collation is marked where
  * its strings hold a character the measured order does not cover, the gap the statement's own key refuses by name
  * (`libc-uncovered`); any other mark is a Mason bug. Once counted (a minute at most), the CASE computes the row from its
- * document, and raises the function's own error (the character named) or answers. Raised by `MARKED_FUNCTIONS`, never
+ * document, and raises the function's own error or answers. Raised by `MARKED_FUNCTIONS`, never
  * DuckDB's `error()`, which is volatile (a window's ORDER BY refuses it: the eager oracle's `rank() OVER (ORDER BY j)`).
  */
 export function markedError(r, libc) {
     if (isOrderRead(r) && libc) {
-        return { code: "0A000", message: "a jsonb compared or sorted here holds a string Mason cannot order as the database's collation does (a combining mark, a Thai or " +
-                "Lao vowel written before its consonant, a private-use or unassigned code point), in a row written since its stored keys were counted; asked again " +
-                "in a minute, the error names the character (Mason gap libc-uncovered)" };
+        return { code: "0A000", message: "a jsonb compared or sorted here holds a string Mason cannot order as the database's collation does, " +
+                "in a row written since its stored keys were counted; asked again " +
+                "in a minute, the statement's own key answers or refuses it (Mason gap libc-uncovered)" };
     }
     return { code: "XX000", message: `a stored value of ${pathText(r)} could not be computed when its row was written (a Mason bug: report it, with mason logs); ` +
             "asked again in a minute, it is computed from the row's document" };
@@ -224,7 +224,8 @@ export function exposes(tree) {
             return false;
         const k = kindOf(v);
         const b = k === null ? v : bodyOf(v);
-        if (k === "SelectStmt" && Array.isArray(b.targetList) && directTables(b.fromClause).length > 0
+        // (an arm of a set operation is a SelectStmt with no wrapper of its own: k is null)
+        if ((k === "SelectStmt" || k === null) && Array.isArray(b.targetList) && directTables(b.fromClause).length > 0
             && b.targetList.some((t) => kindOf(t) === "ResTarget" && star(bodyOf(t).val)))
             return true;
         if (k === "JoinExpr" && b.isNatural === true && directTables([b.larg, b.rarg]).length > 0)

package/dist/src/duck/kept.js

@@ -2,8 +2,8 @@ import { DUCKDB_VERSION, MASON_SCHEMA } from "./store.js";
 import { createHash } from "node:crypto";
 import { canonicalRead, EQUAL_STEPS, HIDDEN_PREFIX, HIDDEN_RE, hiddenName, isEqualRead, isKeyRead, isOrderRead, KEY_NULL_HEX, KEY_RECOMPUTE_HEX, knownSteps, ORDER_STEPS, pathKey, pathText, readOfKey, VISIBLE_CATALOG } from "./kept-route.js";
 import { JSONB_KEY_FUNCTIONS, KEY_VERSION, libcUncovered, STORED_SUFFIX } from "./jsonb-order.js";
+import { TURN_FUNCTION } from "./functions.js";
 import { encodeLibcOrder } from "./libc-order.js";
-import { icuNameOf } from "./rewrite.js";
 /** a path read by this many statements in the request log is stored (a larger table's also needs `KEEP_FILL_SHARE`) */
 export const KEEP_AFTER_STATEMENTS = 3;
 /** a path of a SMALL table (`SMALL_FILL_SECONDS`) read by one statement that took this long (ms, the request log's
@@ -89,7 +89,7 @@ export function storeJsonPathsOf(env) {
     if (v === "off")
         return false;
     throw new Error(`${STORE_JSON_PATHS_SETTING}=${JSON.stringify(env[STORE_JSON_PATHS_SETTING])} is not a setting: on (the default) stores the JSON paths statements keep reading ` +
-        "as columns of the copy, off stores none (and drops those stored, at this start)");
+        "as columns of the synced table, off stores none (and drops those stored, at this start)");
 }
 const ident = (name) => `"${name.replaceAll('"', '""')}"`;
 const literal = (s) => `'${s.replaceAll("'", "''")}'`;
@@ -97,7 +97,7 @@ const tableOf = (source) => { const at = source.indexOf("."); return `${ident(so
 /** the value a stored column takes from its JSON column's `value` (a column reference or an expression): the path's
  *  `->` answer, never NULL (a NULL document's is `KEPT_NULL` too: the column's NULL is a row not stored yet) */
 export function keptValueSql(p, value) {
-    return `coalesce(__mason_json_kept(NULL, ${value}, ${literal(p.steps)}, ${p.either}, ${p.canonical}, false), ${NULL_SQL})`;
+    return `coalesce(__mason_json_kept(${TURN_FUNCTION}(NULL, ${value}), ${value}, ${literal(p.steps)}, ${p.either}, ${p.canonical}, false), ${NULL_SQL})`;
 }
 /**
  * A JSONB'S STORED KEY, from its column's `value` (src/duck/kept-route.ts `ORDER_STEPS`): the key the statement would
@@ -106,9 +106,9 @@ export function keptValueSql(p, value) {
  * `KEY_RECOMPUTE_HEX` where the key raised, which its read computes (and raises) again.
  */
 export function orderKeySql(c, value) {
-    const call = (name, ...args) => `${name}${STORED_SUFFIX}(NULL, ${args.join(", ")})`;
-    const key = c.kind === "bytes" ? call(JSONB_KEY_FUNCTIONS.bytes.key, value)
-        : call(JSONB_KEY_FUNCTIONS[c.kind].key, value, `list_transform(${call(JSONB_KEY_FUNCTIONS[c.kind].strings, value)}, __mason_s -> icu_sort_key(__mason_s, ${literal(icuNameOf(c))}))`);
+    const call = (name, ...args) => `${name}${STORED_SUFFIX}(${TURN_FUNCTION}(NULL, ${value}), ${args.join(", ")})`;
+    const key = c.kind !== "icu" ? call(JSONB_KEY_FUNCTIONS[c.kind].key, value)
+        : call(JSONB_KEY_FUNCTIONS.icu.key, value, `list_transform(${call(JSONB_KEY_FUNCTIONS.icu.strings, value)}, __mason_s -> icu_sort_key(__mason_s, ${literal(c.name)}))`);
     return `CASE WHEN (${value}) IS NULL THEN unhex('${KEY_NULL_HEX}') ELSE coalesce(${key}, unhex('${KEY_RECOMPUTE_HEX}')) END`;
 }
 /**
@@ -124,7 +124,7 @@ export function orderId(c, libc, versions = `k${KEY_VERSION}:${DUCKDB_VERSION}`)
         return `icu:${c.name}:${versions}`;
     if (libc === undefined)
         throw new Error("a libc collation's order was not read (Mason bug: report it)");
-    return `libc:${c.icu}:${createHash("sha256").update(encodeLibcOrder(libc)).digest("hex").slice(0, 16)}:${versions}`;
+    return `libc:${createHash("sha256").update(encodeLibcOrder(libc)).digest("hex").slice(0, 16)}:${versions}`;
 }
 /**
  * THE C KEY'S ID, stored under a libc collation for equality (`EQUAL_STEPS:<id>`, src/duck/kept-route.ts): the C
@@ -381,7 +381,7 @@ export class KeptJsonPaths {
         for (const p of this.paths.values()) {
             if (p.state === "dropped" || p.state === "waiting" || named.has(`${p.source}\u0000${p.name}`))
                 continue;
-            await this.drop(w, p, "its column is not in the copy");
+            await this.drop(w, p, "its column is not in the synced table");
         }
         // a jsonb's stored key is its collation's, told after this (`orderBy`): until then a write leaves it unset and no
         // plan reads it, so a ready one is filled again where unset first — the backfill's sweep alone (its ranges are done)
@@ -482,7 +482,7 @@ export class KeptJsonPaths {
     async swapping(c, source, columns) {
         this.gone = [];
         for (const p of this.physical(source).filter((x) => !columns.includes(x.column))) {
-            const why = `${p.column} is no longer a column of the copy of ${source}`;
+            const why = `${p.column} is no longer a column of the synced table ${source}`;
             await c.run(`UPDATE ${PATHS_TABLE} SET state = 'dropped', why = ${literal(why)} WHERE name = ${literal(p.name)}`);
             this.gone.push({ p, why });
         }
@@ -662,7 +662,7 @@ export class KeptJsonPaths {
                         await this.check(p);
             }
             catch (e) {
-                this.host.log(`stored JSON paths: the review failed, and runs again in ${Math.round(this.reviewMs / 1000)} s (the copies serve as they were): ${e instanceof Error ? e.message : String(e)}`);
+                this.host.log(`stored JSON paths: the review failed, and runs again in ${Math.round(this.reviewMs / 1000)} s (the synced tables serve as they were): ${e instanceof Error ? e.message : String(e)}`);
             }
             finally {
                 this.reviewing = null;
@@ -726,10 +726,10 @@ export class KeptJsonPaths {
             return false;
         const name = hiddenName(read);
         const col = cols.find((c) => c.name === read.column);
-        const why = col === undefined ? `${read.column} is not a column of the copy`
+        const why = col === undefined ? `${read.column} is not a column of the synced table`
             : !/^jsonb?$/.test(col.type) ? `${read.column} is a ${col.type}, not a json or jsonb`
                 : isKeyRead(read) && col.type !== "jsonb" ? `${read.column} is a ${col.type}, which has no order or equality (a jsonb's is stored)`
-                    : cols.some((c) => c.name.toLowerCase() === name) ? `the copy has a column named ${name} of its own` : null;
+                    : cols.some((c) => c.name.toLowerCase() === name) ? `the synced table has a column named ${name} of its own` : null;
         if (why === null)
             return true;
         const key = pathKey(read);
@@ -879,7 +879,9 @@ export class KeptJsonPaths {
         const of = read < marked ? ` (of the first ${read.toLocaleString("en-US")} read, of ${marked.toLocaleString("en-US")})` : "";
         const said = [];
         if (uncovered > 0)
-            said.push(`${uncovered.toLocaleString("en-US")} row(s) hold a character the database's libc order is not measured for, so a sort of them is refused by name (Mason gap libc-uncovered)`);
+            said.push(libc?.unread === true
+                ? `the database's collation is one Mason could not read, so a sort of ${uncovered.toLocaleString("en-US")} row(s) is refused by name (Mason gap libc-locale-unread)`
+                : `${uncovered.toLocaleString("en-US")} row(s) hold a character the database's libc order is not measured for, so a sort of them is refused by name (Mason gap libc-uncovered)`);
         if (read > uncovered)
             said.push(`${(read - uncovered).toLocaleString("en-US")} row(s) its function could not read are computed from their document when read (a Mason bug: report it, with mason logs)`);
         return said.join("; ") + of;