@mason-data/cli 0.7.0 → 0.9.0

package/dist/src/duck/store.js

@@ -16,23 +16,23 @@
  * (src/duck/sync.ts attach), so the store itself opens offline.
  */
 import * as os from "node:os";
-import { setTimeout as sleep } from "node:timers/promises";
 import { DuckDBInstance, version } from "@duckdb/node-api";
 import { Executor } from "./executor.js";
 import { beginStatement, cancelStatement, slotName, statementCancelled } from "./cancel.js";
-import { StatementError } from "../api/postgres/server.js";
-import { duckParameters, missingRelationHint, planDuck, quoteDuckKeywords } from "./plan.js";
+import { StatementError } from "./statement.js";
+import { duckParameters, missingRelationHint, ordersText, planDuck, quoteDuckKeywords } from "./plan.js";
 import { VERIFIED } from "./verified.js";
-import { msSinceCall, registerFunctions, SESSION_VARIABLE } from "./functions.js";
-import { duckCollation } from "./rewrite.js";
-import { decodeLibcOrder, encodeLibcOrder } from "./libc-order.js";
+import { registerFunctions, SESSION_VARIABLE, SETTINGS_VARIABLE } from "./functions.js";
+import { caseMappingOf, CollationResolver, duckCollation, isLibcCollation } from "./collation.js";
+import { decodeLibcOrder, encodeLibcOrder, isCurrentLibcOrder } from "./libc-order.js";
 import { dataRow } from "../api/postgres/protocol.js";
 import { cellText, exactColumn, pgTypedColumn, wireColumn } from "./types.js";
-import { gapMark } from "./gaps.js";
-import { cellsOf, evaluateText, NumericError } from "../lib/pg-numeric.js";
+import { gapIn, gapMark } from "./gaps.js";
+import { cellsOf, evaluateText, fieldOverflow, NumericError } from "../lib/pg-numeric.js";
 import { explainJson, explainText, planNodes, profiled, qualifiedName, scannedTables } from "./explain.js";
 import * as fs from "node:fs";
 import * as path from "node:path";
+import { extensionSettings, findExtensions } from "./extensions.js";
 export const MASON_SCHEMA = "_mason";
 /** the DuckDB the prototype runs (`@duckdb/node-api`'s bundled library) */
 export const DUCKDB_VERSION = version();
@@ -57,25 +57,102 @@ export const DUCKDB_VERSION = version();
  * holds the key under every collation), and a LEFT lateral's unnest of an expression, which computes the expression
  * once with `list_transform(list_value(e), l -> CASE … l … END)[1]` (`rangeFunction`; tests/duck-wire.test.ts). A `->`
  * that is not a lambda function's argument is still the json operator.
+ *
+ * Beside them, the copy's extensions (src/duck/extensions.ts `extensionSettings`): with Mason's package for this
+ * platform, `extension_directory` is that package and nothing is ever downloaded; without one, DuckDB's defaults.
  */
 const INSTANCE_SETTINGS = { late_materialization_max_rows: "10000", force_compression: "zstd", lambda_syntax: "ENABLE_SINGLE_ARROW" };
 /**
  * DUCKDB'S MEMORY, BOUNDED FOR THIS PROCESS: DuckDB's default is 80% of the machine's RAM, in a process
  * that also holds V8's heap and two PGlites, which ends with the system killing every session and the sync together.
- * Half the RAM unless `MASON_DUCKDB_MEMORY` says (`8GB`, `512MiB`); past it a sort, a join or an aggregate spills to
- * `<file>.tmp` (DuckDB's own default, named here so it is the copy's folder), and what cannot spill fails ITS statement
- * with 53200 and the way out (pgErrorOf), the run going on.
+ * Unless `MASON_DUCKDB_MEMORY` says (`8GB`, `512MiB`), half the memory, or what leaves `OUTSIDE_RESERVE_BYTES` beside
+ * it when that is less (`defaultMemoryLimit`); past it a sort, a join or an aggregate spills to `<file>.tmp` (DuckDB's
+ * own default, named here so it is the copy's folder), and what cannot spill fails ITS statement with 53200 and the way
+ * out (pgErrorOf), the run going on.
+ *
+ * THE ROOM OUTSIDE THE LIMIT (issue 301, docs/DESIGN.md "Memory: the room outside the engine's limit"): the limit counts
+ * DuckDB's buffers, never Node's heap, the catalog's PGlite, the pg-eval PGlite nor what DuckDB allocates outside its
+ * buffer manager, and with half of a 1, 1.5, 2 or 3 GiB container that room was too small: the kernel killed the run in
+ * each. Measured with the limit at 256MiB, the process reached 2.3 GiB beside it (the pg-eval PGlite alone ~0.5 GiB of
+ * WebAssembly memory once a statement handed it 1.5 M values, the catalog's ~0.26 GiB). `OUTSIDE_RESERVE_BYTES` is that
+ * peak with a margin; `MEMORY_FLOOR_BYTES` the smallest container the whole path (a first sync, a generated table's
+ * build, writes on the stream, statements through the listener) ran in under this default. Under the floor, with
+ * `MASON_DUCKDB_MEMORY` unset, a start goes on and WARNS (`memoryWarning`: the run's start, mason status, mason doctor),
+ * its statements given MIN_SHARE_BYTES, the least room for them and the most for the rest (Tom's decision on PR 382: a
+ * warning, never a refusal); set, the variable is the operator's word at any size.
+ */
+export const OUTSIDE_RESERVE_BYTES = 2560 * 2 ** 20;
+export const MEMORY_FLOOR_BYTES = 3072 * 2 ** 20;
+/** the statements' share where the reserve leaves less (under the floor, where a start warns): what let the run last
+ *  longest, measured (docs/DESIGN.md): at 2 GiB the whole path ran with it, as with 256MiB, and at 1.5 GiB it answered
+ *  every statement before the kill where 256MiB was killed in the writes; the same statements answered at both, one
+ *  wide aggregate failing alone (53200), and the old quarter, 512MiB, was killed at 2 GiB */
+export const MIN_SHARE_BYTES = 128 * 2 ** 20;
+/**
+ * The default memory limit for `totalBytes` of memory, in whole MiB (mason doctor says it for any machine): half of it,
+ * the page cache keeping the other half, or what leaves OUTSIDE_RESERVE_BYTES beside it when that is less (from 5 GiB
+ * up it is half), never under MIN_SHARE_BYTES (from 2.625 GiB down: under the floor, where a start warns), nor past half.
+ */
+export function defaultMemoryLimit(totalBytes) {
+    const mib = 2 ** 20;
+    const half = Math.floor(totalBytes / 2 / mib), room = Math.floor((totalBytes - OUTSIDE_RESERVE_BYTES) / mib);
+    return `${Math.min(half, Math.max(MIN_SHARE_BYTES / mib, room))}MiB`;
+}
+/** A size as a reader recognises it: GiB to one decimal from 1 GiB up, else MiB. */
+export function memorySize(bytes) {
+    return bytes >= 2 ** 30 ? `${(bytes / 2 ** 30).toFixed(1)} GiB` : `${Math.round(bytes / 2 ** 20)} MiB`;
+}
+/** `MASON_DUCKDB_MEMORY` as it is set: null unset (or set empty, as the run reads it) */
+function memorySet(env) {
+    const v = env.MASON_DUCKDB_MEMORY;
+    return v === undefined || v === "" ? null : v;
+}
+/** `MASON_DUCKDB_MEMORY` that is not an amount: the one text, for the store's open, mason serve, mason start and doctor */
+export function memorySettingWrong(text) {
+    return { message: `MASON_DUCKDB_MEMORY=${JSON.stringify(text)} is not an amount of memory`,
+        why: "it is the memory the synced data's statements may use before they spill to disk",
+        fix: `write it as 8GB or 512MiB, or unset it for the default (${DEFAULT_MEMORY_LIMIT} here: ${DEFAULT_MEMORY_RULE})` };
+}
+/** THE DEFAULT'S RULE IN WORDS, the one text every message, help and guide that states the default says (a fact derived
+ *  in several places: PR 380's first-sync refusal said "half the machine's RAM" after the default had changed);
+ *  `defaultMemoryLimit` is the rule, and `tests/conventions.test.ts` holds src/ to saying it only through this */
+export const DEFAULT_MEMORY_RULE = `half the container's memory limit, else of the machine's RAM, or what leaves ${memorySize(OUTSIDE_RESERVE_BYTES)} ` +
+    `beside it when that is less, never under ${memorySize(MIN_SHARE_BYTES)}`;
+/**
+ * WHAT A START REFUSES OF THE MEMORY (mason serve's one report; mason start and mason doctor take the first):
+ * `MASON_DUCKDB_MEMORY` that is not an amount. A size under the floor is never refused: it is `memoryWarning`'s.
+ */
+export function memoryProblems(env) {
+    const set = memorySet(env);
+    return set !== null && !MEMORY_LIMIT_RE.test(set) ? [memorySettingWrong(set)] : [];
+}
+/**
+ * THE ONE WARNING OF THE MEMORY: `memory` under MEMORY_FLOOR_BYTES while `MASON_DUCKDB_MEMORY` is unset (or not an
+ * amount, which is refused beside it). The start goes on; the run says it at its start and in what mason status reads
+ * (its warnings), and mason doctor's memory line says it, each from this text alone. `memory` is `machineMemory()`.
  */
-/** The default memory limit for a machine with `totalBytes` of RAM: half of it, in MiB (mason doctor says it for any machine). */
-export function defaultMemoryLimit(totalBytes) { return `${Math.floor(totalBytes / 2 / 2 ** 20)}MiB`; }
+export function memoryWarning(memory, env) {
+    const set = memorySet(env);
+    if ((set !== null && MEMORY_LIMIT_RE.test(set)) || memory.bytes >= MEMORY_FLOOR_BYTES)
+        return null;
+    const size = memorySize(memory.bytes);
+    return { message: `${size} of memory is under the ${memorySize(MEMORY_FLOOR_BYTES)} Mason needs`,
+        why: `${size} is ${memory.from === "container" ? "the container's memory limit" : "the machine's RAM"}; ` +
+            `beside what the synced data's statements may use, Mason itself needs up to ${memorySize(OUTSIDE_RESERVE_BYTES)} (measured, with a margin), ` +
+            "and with less the system may stop the whole run, every session with it, when a first sync, a burst of writes or a statement needs the memory",
+        fix: `give it ${memorySize(MEMORY_FLOOR_BYTES)} or more (the container's memory limit), or set MASON_DUCKDB_MEMORY=<n> ` +
+            "to choose the statements' share yourself, the rest left to Mason (mason doctor says what it leaves)" };
+}
+/** a problem or the warning as one line: mason serve's settings report, the run's warning, mason doctor's detail */
+export function memoryProblemText(p) { return `${p.message}: ${p.why} — ${p.fix}`; }
 /**
  * THE MEMORY THIS PROCESS MAY USE: the machine's RAM, or the container's limit where one is set — a cgroup v2
  * `memory.max` or a cgroup v1 `memory.limit_in_bytes` below it. `os.totalmem()` is the HOST's RAM inside a container,
  * so half of it was more than a limited container had, and the kernel killed the run instead of DuckDB spilling
  * (the review of the cut). `read` is the test's.
  */
-export function machineMemoryBytes(read = (f) => fs.readFileSync(f, "utf8"), total = os.totalmem()) {
-    let limit = total;
+export function machineMemory(read = (f) => fs.readFileSync(f, "utf8"), total = os.totalmem()) {
+    let limit = { bytes: total, from: "machine" };
     for (const file of ["/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"]) {
         let text;
         try {
@@ -86,12 +163,35 @@ export function machineMemoryBytes(read = (f) => fs.readFileSync(f, "utf8"), tot
         } // not a cgroup of that version, or not a container: the machine's RAM
         // v2 says `max` for no limit; v1 says a number near 2^63; either way a number below the RAM is the limit
         const n = Number(text);
-        if (text !== "max" && Number.isFinite(n) && n > 0 && n < limit)
-            limit = n;
+        if (text !== "max" && Number.isFinite(n) && n > 0 && n < limit.bytes)
+            limit = { bytes: n, from: "container" };
     }
     return limit;
 }
+/** `machineMemory`'s bytes: the one size the default limit and the floor are read against */
+export function machineMemoryBytes(read, total) { return machineMemory(read, total).bytes; }
 export const DEFAULT_MEMORY_LIMIT = defaultMemoryLimit(machineMemoryBytes());
+/**
+ * WHAT STATEMENTS MAY SPILL (issue 105: a 100-row preview's sort took every free byte of a volume, and the sync's room
+ * with it). DuckDB's default ceiling is "90% of available disk space", and what it answered when the disk ran out first
+ * was `XX000` with a path. Here the ceiling is HALF of the room statements have, what is free where the copy is and what
+ * they have spilled there (the disk's free bytes exclude the spill, so a ceiling of the free bytes alone would grow with
+ * each read as the spill grows, and end near the whole disk). The sync's writes keep the other half whatever statements
+ * do, and it follows the disk (`DuckStore.keepSpillCeiling`, every `SPILL_REFRESH_MS`): a copy that grows narrows it.
+ * `MASON_SPILL_LIMIT` sets the amount instead, never past 90% of that room. Spilled files are shared by the statements
+ * running, so two at once share the ceiling. Past it the statement that needed the room fails alone, 53400 as past
+ * Postgres's temp_file_limit, with the way out (pgErrorOf), and its files are gone with it; a disk that is really full
+ * is 53100. A sizing decision reads `store.spillLimitBytes`, as it reads `store.memoryLimitBytes`.
+ */
+const SPILL_FREE_SHARE = 0.5, SPILL_FREE_MOST = 0.9;
+const SPILL_REFRESH_MS = 15_000;
+export function spillCeiling(freeBytes, spilledBytes, setBytes) {
+    const room = freeBytes + spilledBytes;
+    return Math.max(2 ** 20, Math.floor(setBytes === null ? room * SPILL_FREE_SHARE : Math.min(setBytes, room * SPILL_FREE_MOST)));
+}
+/** the bytes free where `dir` is: the one reading of it (the sync's disk check, the spill's ceiling) */
+export function freeBytesAt(dir) { const st = fs.statfsSync(dir); return st.bavail * st.bsize; }
+const spillSetting = (bytes) => `${Math.floor(bytes / 2 ** 20)}MiB`;
 export const MEMORY_LIMIT_RE = /^\s*(\d+(?:\.\d+)?)\s*(KB|MB|GB|TB|KiB|MiB|GiB|TiB)\s*$/i;
 /** DuckDB's memory limit in bytes, as DuckDB reads the text: KB / MB / GB / TB are powers of 1000, KiB / MiB / GiB / TiB of 1024 */
 export function memoryLimitBytes(text) {
@@ -117,8 +217,8 @@ export async function openDuck(file, settings) {
     catch (e) {
         const m = e.message;
         throw new Error(/lock/i.test(m)
-            ? `DuckDB could not open ${file}: another process holds it (${m}). Is another mason start running for this project? mason status says`
-            : `DuckDB could not open ${file}: ${m}. mason destroy deletes the copy, and the next mason start copies again`);
+            ? `Mason could not open the synced data ${file}: another process holds it (${m}). Is another mason start running for this project? mason status says`
+            : `Mason could not open the synced data ${file}: ${m}. mason destroy deletes the synced data, and the next mason start syncs again`);
     }
     const writer = await instance.connect();
     await writer.run(`CREATE SCHEMA IF NOT EXISTS ${MASON_SCHEMA}`);
@@ -136,30 +236,54 @@ export class DuckStore {
     file;
     /** the memory limit every connection shares, in bytes */
     memoryLimitBytes;
+    /** what statements may spill now, in bytes (`spillCeiling`): it follows the disk */
+    spillLimitBytes;
+    spillSet;
+    diskFree;
+    spillTimer;
+    spillLane = null;
+    spillKept = null;
+    log;
     executor;
     closed = null;
     closeGraceMs;
-    constructor(file, executor, memoryLimit, closeGraceMs) {
+    /** where the copy's extensions come from: Mason's package for this platform, or DuckDB's install (src/duck/extensions.ts) */
+    extensions;
+    constructor(file, executor, memoryLimit, extensions, spill, log, closeGraceMs) {
         this.file = file;
         this.executor = executor;
         this.memoryLimitBytes = memoryLimit;
+        this.extensions = extensions;
         this.closeGraceMs = closeGraceMs;
+        this.spillLimitBytes = spill.limit;
+        this.spillSet = spill.set;
+        this.diskFree = spill.diskFree;
+        this.log = log;
+        this.spillTimer = setInterval(() => { void this.keepSpillCeiling(); }, SPILL_REFRESH_MS);
+        this.spillTimer.unref();
         this.writer = executor.writer();
         this.instance = { connect: () => executor.connect() };
     }
     /** `silentMs` and `closeGraceMs`: the executor's bounds (src/duck/executor.ts SILENT_MS, CLOSE_GRACE_MS), shorter only
-     *  in a test; `workerFile`, the executor worker's module, another only in a test */
+     *  in a test; `workerFile`, the executor worker's module, another only in a test; `extensions`, this machine's
+     *  (`findExtensions`) unless a test names a package */
     static async open(file, log, options = {}) {
+        const extensions = options.extensions ?? findExtensions();
         const memoryLimit = options.memoryLimit ?? DEFAULT_MEMORY_LIMIT;
         if (!MEMORY_LIMIT_RE.test(memoryLimit))
-            throw new Error(`MASON_DUCKDB_MEMORY=${JSON.stringify(memoryLimit)} is not an amount of memory: write it as 8GB or 512MiB (unset, half the machine's RAM: ${DEFAULT_MEMORY_LIMIT})`);
+            throw new Error(memoryProblemText(memorySettingWrong(memoryLimit)));
+        if (options.spillLimit !== undefined && !MEMORY_LIMIT_RE.test(options.spillLimit))
+            throw new Error(`MASON_SPILL_LIMIT=${JSON.stringify(options.spillLimit)} is not an amount of disk: write it as 40GB or 512MiB (unset, half of what is free on the disk)`);
+        const spillSet = options.spillLimit === undefined ? null : memoryLimitBytes(options.spillLimit);
+        const diskFree = options.diskFree ?? freeBytesAt;
         const where = path.resolve(file);
+        const spillLimit = spillCeiling(diskFree(path.dirname(where)), 0, spillSet);
         const before = opened.get(where);
         if (before !== undefined && before.closed === null)
-            throw new Error(`DuckDB could not open ${file}: this process has it open already (Mason bug: report it)`);
+            throw new Error(`Mason could not open the synced data ${file}: this process has it open already (Mason bug: report it)`);
         // a copy whose thread was given up on may still hold the file: a second instance of it would lose writes
         if (before?.abandoned != null)
-            throw new Error(`DuckDB could not open ${file}: this process gave up on the copy's thread before (${before.abandoned}), which may still hold it; start mason again (a new process opens it)`);
+            throw new Error(`Mason could not open the synced data ${file}: this process gave up on the engine's thread before (${before.abandoned}), which may still hold it; start mason again (a new process opens it)`);
         const mine = { closed: null, abandoned: null };
         opened.set(where, mine);
         let executor;
@@ -167,7 +291,7 @@ export class DuckStore {
             if (before !== undefined)
                 await before.closed;
             // every connection's, the sync's and each session's
-            executor = await Executor.start({ file, settings: { ...INSTANCE_SETTINGS, memory_limit: memoryLimit, temp_directory: `${file}.tmp` }, log,
+            executor = await Executor.start({ file, settings: { ...INSTANCE_SETTINGS, ...extensionSettings(extensions), memory_limit: memoryLimit, temp_directory: `${file}.tmp`, max_temp_directory_size: spillSetting(spillLimit) }, log,
                 ...(options.silentMs !== undefined ? { silentMs: options.silentMs } : {}), ...(options.workerFile !== undefined ? { workerFile: options.workerFile } : {}) });
         }
         catch (e) {
@@ -175,7 +299,7 @@ export class DuckStore {
                 opened.delete(where);
             throw e;
         }
-        const store = new DuckStore(file, executor, memoryLimitBytes(memoryLimit), options.closeGraceMs);
+        const store = new DuckStore(file, executor, memoryLimitBytes(memoryLimit), extensions, { limit: spillLimit, set: spillSet, diskFree }, log, options.closeGraceMs);
         // closed: the file may be opened again — unless its thread was given up on (`abandoned`), which keeps it here
         store.closing = (p) => { mine.closed = p; void p.then(() => { if (executor.abandoned !== null)
             mine.abandoned = executor.abandoned;
@@ -184,23 +308,63 @@ export class DuckStore {
         return store;
     }
     closing = () => undefined;
-    /** called once if the copy's thread dies while the run uses it (src/duck/executor.ts): the run exits with the reason */
+    /**
+     * THE SPILL'S CEILING MOVED WITH THE DISK (`spillCeiling`): the bytes free and spilled now, set on the instance on a
+     * connection of its own, never the writer's. One at a time; a move under 1 % is none. DuckDB refuses a ceiling under
+     * what is spilled at that moment (more may have spilled since the read): it stays where it was until the next turn.
+     * Any other failure is said; the ceiling stands, which is the last one set.
+     */
+    keepSpillCeiling() {
+        if (this.closed !== null)
+            return Promise.resolve();
+        this.spillKept ??= this.spillTurn().catch((e) => {
+            if (this.closed === null)
+                this.log(`the spill's ceiling was not moved: ${e instanceof Error ? e.message : String(e)}`);
+        }).finally(() => { this.spillKept = null; });
+        return this.spillKept;
+    }
+    async spillTurn() {
+        let c;
+        // a connection that failed is asked for again next turn, never kept as the answer
+        try {
+            c = await (this.spillLane ??= this.executor.connect());
+        }
+        catch (e) {
+            this.spillLane = null;
+            throw e;
+        }
+        const spilled = Number((await c.runAndReadAll("SELECT coalesce(sum(size), 0) FROM duckdb_temporary_files()")).getRowsJS()[0][0]);
+        const next = spillCeiling(this.diskFree(path.dirname(path.resolve(this.file))), spilled, this.spillSet);
+        if (Math.abs(next - this.spillLimitBytes) < this.spillLimitBytes * 0.01)
+            return;
+        try {
+            await c.run(`SET GLOBAL max_temp_directory_size = '${spillSetting(next)}'`);
+        }
+        catch (e) {
+            if (e instanceof Error && e.message.includes("failed to adjust"))
+                return;
+            throw e;
+        }
+        this.spillLimitBytes = next;
+    }
+    /** called once if the engine's thread dies while the run uses it (src/duck/executor.ts): the run exits with the reason */
     onCrash(f) { this.executor.onCrash(f); }
-    /** how many vectors each Mason function was called with so far, on the copy's thread (a test's, a measurement's) */
+    /** how many vectors each Mason function was called with so far, on the engine's thread (a test's, a measurement's) */
     functionCalls() { return this.executor.functionCalls(); }
-    /** how many rows each Mason function was handed so far, on the copy's thread (a plan's cost in calls) */
+    /** how many rows each Mason function was handed so far, on the engine's thread (a plan's cost in calls) */
     functionRows() { return this.executor.functionRows(); }
     /** a connection for one Postgres session (on the executor worker: SessionRunner) */
     runner() { return this.executor.runner(); }
     /**
      * The source's order of text under a libc collation (src/duck/libc-order.ts), kept in the copy: the one recorded
-     * for this collation and version, else `read` from the source (a few seconds) and recorded. `read: true` when it was.
+     * for this collation and version, in this release's form, else `read` from the source (a few seconds) and recorded.
+     * `read: true` when it was.
      */
     async libcOrder(collate, version, read) {
         await this.writer.run(`CREATE SCHEMA IF NOT EXISTS ${MASON_SCHEMA}`);
         await this.writer.run(`CREATE TABLE IF NOT EXISTS ${MASON_SCHEMA}.libc_order (collation_name VARCHAR, collation_version VARCHAR, data VARCHAR)`);
         const kept = (await this.writer.runAndReadAll(`SELECT data FROM ${MASON_SCHEMA}.libc_order WHERE collation_name = $1 AND collation_version IS NOT DISTINCT FROM $2`, [collate, version])).getRowsJS();
-        if (kept.length > 0)
+        if (kept.length > 0 && isCurrentLibcOrder(String(kept[0][0])))
             return { order: decodeLibcOrder(String(kept[0][0])), read: false };
         const order = await read();
         // one order per collation: a version that moved replaces the last one's
@@ -212,22 +376,50 @@ export class DuckStore {
      *  its text sorted as the SOURCE sorts it (`collation`, pg_database's; absent, by bytes; a libc one by `libc`, its
      *  order as `libcOrder` read it), refusing what is not `verified` (null: nothing is refused for it, the oracle's
      *  recording run alone). */
-    async listener(collation, verified = VERIFIED, libc, copies = () => [], kept) {
+    async listener(collation, verified = VERIFIED, libc, copies = () => [], kept, more = {}) {
         const known = new Set((await this.writer.runAndReadAll("SELECT collname FROM pragma_collations()")).getRowsJson().map((r) => String(r[0])));
         quoteDuckKeywords((await this.writer.runAndReadAll("SELECT keyword_name FROM duckdb_keywords() WHERE keyword_category <> 'unreserved'")).getRowsJson().map((r) => String(r[0])));
-        const c = collation === undefined ? { kind: "bytes" } : duckCollation(collation.provider, collation.collate, collation.locale, known);
+        const named = collation === undefined ? { kind: "bytes" } : duckCollation(collation.provider, collation.collate, collation.locale, known);
+        // (a libc locale whose strcoll is byte order, musl's or macOS's: src/duck/libc-order.ts)
+        const c = named.kind === "libc" && libc?.bytes === true ? { kind: "bytes" } : named;
         if (c.kind === "libc") {
             if (libc === undefined)
                 throw new Error(`the database orders text by the libc collation ${collation.collate}, and its order was not read (Mason bug: report it)`);
-            await this.executor.registerLibcOrder(libc);
+            await this.executor.registerLibcOrder(c.collate, libc, true);
         }
+        // each other libc collation read at the start (a synced column's): its key's order, by its collcollate
+        for (const [collate, order] of more.libcOrders ?? [])
+            if (!order.bytes)
+                await this.executor.registerLibcOrder(collate, order, false);
+        // a libc collation named first by a statement: read from the source on its first use, kept in the copy, then the
+        // engine told its order before the plan that needs it is made (src/duck/collation.ts)
+        const readLibc = more.readLibc;
+        // the database's case mapping is its CTYPE's (datctype; ICU's or builtin's locale), never its order's
+        const ctype = collation === undefined ? null : collation.provider === "c" ? collation.ctype ?? collation.collate : collation.locale ?? collation.collate;
+        const resolver = new CollationResolver({ database: c, databaseCollate: collation !== undefined && isLibcCollation(collation.provider, collation.collate) ? collation.collate : null,
+            ...(collation === undefined ? {} : { databaseCase: { mapping: caseMappingOf(collation.provider, ctype, collation.locale), ctype: ctype ?? "" } }),
+            source: more.collations ?? null, known, libc: more.libcOrders ?? new Map(),
+            readLibc: readLibc === undefined ? null : async (sc) => {
+                const r = await this.libcOrder(sc.collate ?? "C", sc.version ?? null, () => readLibc(sc));
+                if (!r.order.bytes)
+                    await this.executor.registerLibcOrder(sc.collate ?? "C", r.order, false);
+                return r.order;
+            } });
+        // the database's own order, held where a plan reads each order by its collcollate (src/duck/libc-bound.ts)
+        if (c.kind === "libc" && libc !== undefined)
+            resolver.hold(c.collate, libc);
         // a jsonb's stored key is this collation's (src/duck/kept.ts `orderBy`)
         await kept?.orderBy(c, c.kind === "libc" ? libc : undefined);
         return {
             collation: c,
+            collations: resolver,
             open: () => this.runner(),
-            plan: (catalog, text, stmt, base, paramOids, searchPath, timeZone, userTypes, wireBound) => planDuck(catalog, text, stmt, base, paramOids, searchPath, timeZone, c, verified, userTypes, copies, 0, kept ?? null, wireBound ?? false),
+            plan: (catalog, text, stmt, base, paramOids, under, userTypes, wireBound, grants, relations, outputs) => planDuck({ catalog, text, stmt, base, paramOids, searchPath: under.searchPath, timeZone: under.timeZone, dateStyle: under.dateStyle, collations: resolver, verified, userTypes, copies, kept: kept ?? null, wireBound: wireBound ?? false, grants, ...(relations === undefined ? {} : { relations }), outputs: outputs ?? false }),
             parameters: duckParameters,
+            // a SELECT of no table that orders text under a collation that is not byte order: the synced data's (issue 122);
+            // one that maps case (upper, lower, initcap: issue 406) is the synced data's where its run answers it (the
+            // listener's `answersCase`), else the catalog's, as before
+            ordersText: (catalog, text, paramOids, under, bindable) => ordersText({ catalog, text, paramOids, under, collations: resolver, bindable }),
             // the sync's state, which EXPLAIN says how each copy is kept current from (src/duck/sync.ts KeptCurrent)
             copies,
             // a stored JSON path ready or gone (src/duck/kept.ts): the plans kept are made again
@@ -238,6 +430,7 @@ export class DuckStore {
      *  (a second open of the file waits for it), or once the executor's grace has passed whatever the thread does
      *  (src/duck/executor.ts CLOSE_GRACE_MS); never rejects — a failure is said in the log. */
     close() {
+        clearInterval(this.spillTimer);
         if (this.closed === null) {
             this.closed = this.executor.close(this.closeGraceMs);
             this.closing(this.closed);
@@ -273,22 +466,35 @@ export function pgErrorOf(err, sql) {
     const full = err instanceof Error ? err.message : String(err);
     const kind = /^([A-Za-z ]+) Error: /.exec(full)?.[1] ?? "";
     const message = full.replace(/^[A-Za-z ]+ Error: /, "").replace(/\n\n?LINE \d+:[\s\S]*$/, "").trim();
+    // NO ROOM TO SPILL (issue 105): the ceiling, or the disk itself before it. Postgres's class for a full disk, in
+    // Mason's words: the engine's text names a path and a setting no client of Mason has, so it stays out of the answer
+    // Read by the engine's own kind and sentence: a statement's literal may hold the same words in any other error
+    const ceiling = kind === "Out of Memory" && /This limit was set by the 'max_temp_directory_size' setting/.test(full);
+    if (ceiling || (kind === "IO" && /No space left on device/.test(full))) {
+        return new StatementError({ code: ceiling ? "53400" : "53100", message: ceiling ? "could not write to temporary file: this statement needs more room on disk than statements may take"
+                : "could not write to temporary file: No space left on device",
+            hint: "ask for less at once (a filter, a LIMIT, fewer or narrower columns, a coarser GROUP BY), free disk where the synced data is, or raise MASON_SPILL_LIMIT (40GB; by default half of what is free) and start mason again; nothing else stopped" }, sql, "spill-limit");
+    }
     const pg = postgresWording(kind, message);
+    // a refusal raised inside DuckDB (a Mason function's, the rewrite's error() call) crossed it as its text alone: the
+    // entry its text names (gapIn), read here, the one place that text becomes a statement's error
+    // (Mason's own refusal keeps the engine's text out of the DETAIL: it may quote the row, tests/conventions.test.ts)
+    // (a Mason function's error that carries Postgres's own DETAIL sends it as Postgres does, apart from the message)
     if (pg !== null)
-        return new StatementError({ ...pg, detail: `DuckDB: ${message}` }, sql);
+        return new StatementError({ code: pg.code, message: pg.message, ...(pg.code === "0A000" ? {} : { detail: pg.detail ?? `the engine: ${message}` }) }, sql, pg.gap ?? (pg.code === "0A000" ? gapIn(pg.message) : null));
     const fn = /(?:Scalar|Aggregate|Table) Function with name "?([A-Za-z_][\w]*)"? does not exist/.exec(message)?.[1];
     if (fn !== undefined) {
         // what reaches DuckDB was verified against Postgres (src/duck/verified.ts), so a function DuckDB lacks is a gap in
         // Mason's rewrite, never something for the client to write in DuckDB's dialect
-        return new StatementError({ code: "0A000", message: `function ${fn}() is not supported: Mason answers SQL with DuckDB, which has no ${fn}`,
-            hint: "this is a gap in Mason's translation: report it with the statement (docs/SQL-SUPPORT.md lists what is known)", detail: `DuckDB: ${message}` });
+        return new StatementError({ code: "0A000", message: `function ${fn}() is not supported: Mason's engine has no ${fn}`,
+            hint: "this is a gap in Mason's translation: report it with the statement (docs/SQL-SUPPORT.md lists what is known)", detail: `the engine: ${message}` });
     }
     const table = /Table with name "?([^"!\s]+)"? does not exist/.exec(message)?.[1];
     if (table !== undefined) {
         // DuckDB's miss comes after the catalog held the table (the plan passed): one dropped as no longer added, whose way
         // back is `mason add`, so no copy state is read
         return new StatementError({ code: "42P01", message: `relation "${table}" does not exist`,
-            hint: missingRelationHint(table, []), detail: `DuckDB: ${message}` });
+            hint: missingRelationHint(table, []), detail: `the engine: ${message}` });
     }
     const code = kind === "Parser" ? "42601"
         : kind === "Catalog" ? "42P01"
@@ -304,17 +510,20 @@ export function pgErrorOf(err, sql) {
                                     : kind === "Out of Range" ? "22003"
                                         : kind === "Interrupt" || /INTERRUPT/i.test(full) ? "57014"
                                             : kind === "Out of Memory" ? "53200"
-                                                : kind === "Not implemented" ? "0A000"
-                                                    : kind === "Invalid Input" ? "22023"
-                                                        : "XX000";
+                                                // a commit that could not pin or allocate a block is the same limit reached, under the transaction's own kind
+                                                // (measured on DuckDB 2.0.0: "Failed to commit: failed to pin block of size 1.0 MiB (365.7 MiB/366.2 MiB used)")
+                                                : kind === "TransactionContext" && /^Failed to commit: failed to (?:pin block|allocate data) of size .* used\)/.test(message) ? "53200"
+                                                    : kind === "Not implemented" ? "0A000"
+                                                        : kind === "Invalid Input" ? "22023"
+                                                            : "XX000";
     return new StatementError({ code, message: code === "57014" ? "canceling statement due to user request"
-            : code === "53200" ? "out of memory: this statement needs more than the memory Mason's DuckDB may use, in a part that cannot spill to disk" : message,
+            : code === "53200" ? "out of memory: this statement needs more than the memory Mason's engine may use, in a part that cannot spill to disk" : message,
         // DuckDB's own advice is SET, which a client of Mason cannot do: Mason's way out instead
         ...(code === "53200" ? { hint: "ask for less at once (a filter, a LIMIT, fewer or narrower columns, a coarser GROUP BY), or raise MASON_DUCKDB_MEMORY (8GB, 24GB) and start mason again; nothing else stopped" } : {}),
         // what Mason ran (the statement as rewritten for DuckDB, src/duck/rewrite.ts): a report of the error needs it
-        detail: `DuckDB${kind ? ` ${kind} Error` : ""} (Mason answers SQL with DuckDB): ${message}${sql === "" ? "" : `; DuckDB ran: ${oneLine(sql, 2000)}`}`,
+        detail: `the engine${kind ? ` ${kind} error` : ""}: ${message}${sql === "" ? "" : `; Mason ran: ${oneLine(sql, 2000)}`}`,
         // Postgres accepted the GROUP BY (it analysed the statement first): DuckDB refusing it is Mason's rewrite's gap
-        ...(code === "42803" ? { hint: "Postgres accepts this GROUP BY and Mason's rewrite did not carry it to DuckDB: report it with the statement; adding the column to the GROUP BY answers meanwhile" } : {}),
+        ...(code === "42803" ? { hint: "Postgres accepts this GROUP BY and Mason's rewrite did not carry it to the engine: report it with the statement; adding the column to the GROUP BY answers meanwhile" } : {}),
     }, sql);
 }
 /** The errors DuckDB raises for what Postgres raises too, in Postgres's code and words (a tool or a person matches them). */
@@ -335,16 +544,23 @@ function postgresWording(kind, message) {
     const field = /^Date out of range: (.*)$/.exec(message);
     if (field)
         return { code: "22008", message: `date field value out of range: ${field[1]}` };
-    // a cast to numeric(p, s) of a value with more than p - s integer digits
-    if (/Casting value "[^"]*" to type DECIMAL\(\d+,\s*\d+\) failed: value is out of range/.test(message))
-        return { code: "22003", message: "numeric field overflow" };
     // an exact sum past DuckDB's 38 digits (a text read as numeric is summed as a DECIMAL(38,18), src/duck/rewrite.ts summed)
     if (/^Overflow in HUGEINT addition\b/.test(message))
-        return { code: "0A000", message: `a sum past DuckDB's 38 digits is not supported by Mason (${gapMark("numeric-exact-sum")})` };
+        return { code: "0A000", message: `a sum past the engine's 38 digits is not supported by Mason (${gapMark("numeric-exact-sum")})`, gap: "numeric-exact-sum" };
     // a numeric(p, s) read from a text holding NaN or Infinity (src/duck/rewrite.ts pgInput): DuckDB's decimal has none
-    const special = /Could not convert string ['"](NaN|-?Infinity)['"] to DECIMAL/.exec(message);
-    if (special)
-        return { code: "0A000", message: `a numeric ${special[1]} read from a text is not supported by Mason (${gapMark("numeric-special")})` };
+    if (/Could not convert string ['"](NaN|-?Infinity)['"] to DECIMAL/.test(message))
+        return { code: "0A000", message: `a numeric NaN or Infinity read from a text is not supported by Mason (${gapMark("numeric-special")})`, gap: "numeric-special" };
+    // A NUMERIC PAST ITS DIGITS, in every spelling the engine gives (a generated table's run stores one, src/duck/generated.ts
+    // failureWords): a cast to numeric(p, s), of a value or a text, with more than p - s digits before the point, said
+    // with Postgres's DETAIL (and a number constant that does not fit: `12345::numeric(3)`, DuckDB's "Could not cast value
+    // 12345 to DECIMAL(3,0)"); past the 38 digits the engine computes a numeric with, by an operation, a ROUND or a cast
+    const typmod = /(?:Casting value "[^"]*" to type|Could not convert string ['"].*['"] to|^Could not cast value \S+ to) DECIMAL\((\d+),\s*(\d+)\)/.exec(message);
+    if (typmod !== null && Number(typmod[1]) < 38) {
+        return { code: "22003", ...fieldOverflow(Number(typmod[1]), Number(typmod[2])) };
+    }
+    if (typmod !== null || /Overflow in (?:\w+ )?(?:addition|subtraction|multiplication|negation|ROUND|cast)\b.*DECIMAL|Overflow in HUGEINT \w+|out of range for (?:the desired )?DECIMAL|Could not cast value .* to DECIMAL/i.test(message)) {
+        return { code: "22003", message: "numeric field overflow", detail: "The value has more than the 38 digits Mason's engine computes a numeric with." };
+    }
     const conv = /Could not convert string '((?:[^']|'')*)' to (INT16|INT32|INT64|DOUBLE|FLOAT|DATE|TIMESTAMP|UUID|BOOL)/.exec(message);
     if (conv) {
         const t = { INT16: "smallint", INT32: "integer", INT64: "bigint", DOUBLE: "double precision", FLOAT: "real", DATE: "date", TIMESTAMP: "timestamp", UUID: "uuid", BOOL: "boolean" }[conv[2]];
@@ -352,9 +568,10 @@ function postgresWording(kind, message) {
     }
     // a Mason function's own error (src/duck/functions.ts): Postgres's words after the prefix
     // Postgres's own error, from its own function (src/duck/pg-eval.ts): its SQLSTATE, said
-    const pg = /^mason\[([0-9A-Z]{5})\]: (.*)$/s.exec(message);
+    // (and its DETAIL on a line of its own, src/duck/functions.ts `raised`)
+    const pg = /^mason\[([0-9A-Z]{5})\]: (.*?)(?:\nDETAIL: (.*))?$/s.exec(message);
     if (pg)
-        return { code: pg[1], message: pg[2] };
+        return { code: pg[1], message: pg[2], ...(pg[3] === undefined ? {} : { detail: pg[3] }) };
     const own = /^mason: (.*)$/s.exec(message);
     if (own)
         return { code: /^cannot |argument list|key must not be null/.test(own[1]) ? "22023" : /^invalid input syntax/.test(own[1]) ? "22P02" : / is not supported| not supported for type /.test(own[1]) ? "0A000" : "XX000", message: own[1] };
@@ -363,12 +580,55 @@ function postgresWording(kind, message) {
         return { code: "22P02", message };
     return null;
 }
-/** how long no call of a session must have been answered before a result of its that did not end is closed: a task
- *  waiting for a call is answered within a turn of the worker's loop (microseconds when it is free), and the work a task
- *  does between two calls is one vector's (2,048 rows) — milliseconds; 50 ms leaves an order of magnitude over both */
-const CALLS_QUIET_MS = 50;
-/** when the wait for a statement's calls to stop is said in the log */
-const CALLS_SLOW_MS = 5_000;
+export class SessionSetup {
+    conn;
+    zone = null;
+    path = null;
+    settings = null;
+    constructor(conn) { this.conn = conn; }
+    /** `name`: what the Mason functions are passed as the session: a slot of the cancel table (`s<n>`, src/duck/cancel.ts)
+     *  for a wire session; any other name is never cancelled through the table (a run is interrupted instead). Set up as a
+     *  session with no role is: UTC, under `public`, no settings, until `settle` says otherwise. */
+    static async open(conn, name) {
+        // NULLs sort as Postgres sorts them: last ascending, FIRST descending (DuckDB's default is last both ways). An
+        // integer division is the rewrite's `//` where Postgres's operator is integer division (src/duck/rewrite.ts), so
+        // DuckDB's `/` stays the float division every other `/` is
+        await conn.run("SET default_null_order = 'nulls_last_on_asc_first_on_desc'");
+        // every Mason function's first argument (src/duck/functions.ts "every call carries its session")
+        await conn.run(`SET VARIABLE ${SESSION_VARIABLE} = ${quoteLiteral(name)}`);
+        const setup = new SessionSetup(conn);
+        await setup.settle(NO_ROLE_SESSION);
+        return setup;
+    }
+    /** the session's zone, search_path and settings, each set when it changed: before each statement */
+    async settle(session) {
+        if (session.timeZone !== this.zone) {
+            await this.conn.run(`SET TimeZone = ${quoteLiteral(session.timeZone)}`);
+            this.zone = session.timeZone;
+        }
+        // Postgres skips a schema of the path that does not exist; DuckDB refuses the SET (and with it every statement):
+        // the path is the schemas DuckDB has (a table added later in a new schema is found at the next statement)
+        const have = new Set((await this.conn.runAndReadAll("SELECT schema_name FROM duckdb_schemas() WHERE database_name = current_database()")).getRowsJS().map(([n]) => n));
+        const wanted = duckSearchPath(session.searchPath).split(",").filter((n) => have.has(n));
+        const path = (wanted.length > 0 ? wanted : ["main"]).join(",");
+        if (path !== this.path) {
+            await this.conn.run(`SET search_path = ${quoteLiteral(path)}`);
+            this.path = path;
+        }
+        // what `current_setting(name)` reads (src/duck/rewrite.ts `settingRead`): the session's settings, as one map
+        const settings = JSON.stringify(session.settings);
+        if (settings !== this.settings) {
+            const entries = Object.entries(session.settings).map(([k, v]) => `${quoteLiteral(k)}: ${quoteLiteral(v)}`).join(", ");
+            await this.conn.run(`SET VARIABLE ${SETTINGS_VARIABLE} = MAP {${entries}}::MAP(VARCHAR, VARCHAR)`);
+            this.settings = settings;
+        }
+    }
+}
+/** a session with no role, as a generated table is planned and run (src/duck/generated-plan.ts): UTC, under `public`, no
+ *  settings (what reads one is refused there: `current_setting` is "it reads a session") */
+export const NO_ROLE_SESSION = { timeZone: "UTC", searchPath: "public", dateStyle: "ISO, MDY", settings: {} };
+/** when the wait for a statement to end is said in the log */
+const ENDING_SLOW_MS = 5_000;
 /** One wire session's DuckDB connection, with the session's zone and search_path applied before each statement. Runs
  *  on the executor worker (src/duck/executor-worker.ts), its statements asked by the listener's RemoteRunner. */
 export class SessionRunner {
@@ -386,40 +646,20 @@ export class SessionRunner {
     openAnswer = null;
     conn;
     log;
-    zone = null;
-    path = null;
-    constructor(conn, log, slot, cancels) {
+    setup;
+    constructor(conn, log, slot, cancels, setup) {
         this.conn = conn;
         this.log = log;
         this.slot = slot;
         this.cancels = cancels;
+        this.setup = setup;
     }
     /** a connection for one Postgres session, in the cancel table's `slot` */
     static async open(instance, log, slot, cancels) {
         const conn = await instance.connect();
-        // NULLs sort as Postgres sorts them: last ascending, FIRST descending (DuckDB's default is last both ways). An
-        // integer division is the rewrite's `//` where Postgres's operator is integer division (src/duck/rewrite.ts), so
-        // DuckDB's `/` stays the float division every other `/` is
-        await conn.run("SET default_null_order = 'nulls_last_on_asc_first_on_desc'");
-        // every Mason function's first argument (src/duck/functions.ts "every call carries its session")
-        await conn.run(`SET VARIABLE ${SESSION_VARIABLE} = ${quoteLiteral(slotName(slot))}`);
-        return new SessionRunner(conn, log, slot, cancels);
-    }
-    async settle(session) {
-        if (session.timeZone !== this.zone) {
-            await this.conn.run(`SET TimeZone = ${quoteLiteral(session.timeZone)}`);
-            this.zone = session.timeZone;
-        }
-        // Postgres skips a schema of the path that does not exist; DuckDB refuses the SET (and with it every statement):
-        // the path is the schemas DuckDB has (a table added later in a new schema is found at the next statement)
-        const have = new Set((await this.conn.runAndReadAll("SELECT schema_name FROM duckdb_schemas() WHERE database_name = current_database()")).getRowsJS().map(([n]) => n));
-        const wanted = duckSearchPath(session.searchPath).split(",").filter((n) => have.has(n));
-        const path = (wanted.length > 0 ? wanted : ["main"]).join(",");
-        if (path !== this.path) {
-            await this.conn.run(`SET search_path = ${quoteLiteral(path)}`);
-            this.path = path;
-        }
+        return new SessionRunner(conn, log, slot, cancels, await SessionSetup.open(conn, slotName(slot)));
     }
+    async settle(session) { await this.setup.settle(session); }
     async prepare(sql) {
         try {
             return await this.conn.prepare(sql);
@@ -478,13 +718,10 @@ export class SessionRunner {
         const p = await this.prepare(sql);
         timing.prepareMs = performance.now() - t0;
         // HOW A STATEMENT IS CANCELLED (src/duck/functions.ts "every call carries its session"; the test
-        // server's hung process). One that calls no Mason function is interrupted, and its result closed at once: the
-        // close is what cancels the executor's tasks, and a close put off lets the session's next statement take the
-        // connection first, after which the close waits for the cancelled query to run to its end (38 s, in
-        // tests/duck-wire.test.ts "statement_timeout"). One that does is cancelled through its session — its next call
-        // raises — AND interrupted (its tasks stop at their next chunk, so a call-free rest past its last call stops too),
-        // and its result is closed only once DuckDB has handed it back (the fetch's own path, or `drain`) and its calls
-        // have stopped (`cleanUp`: a task still inside a call waits for this thread, which a close would hold). Whether it calls
+        // server's hung process). It is interrupted — its tasks stop at their next chunk — and, if it calls a Mason
+        // function, cancelled through its session as well: its next call raises, so a task of it does not call on. Its
+        // result is closed only once DuckDB ended the query, on a thread that is not this one (`cleanUp`: a task still
+        // inside a call waits for this thread, which a close would hold). Whether it calls
         // one is the PLAN's (src/duck/plan.ts PlannedSql `functions`, read off the rewritten tree), never a match on its
         // text: a stored JSON path's column and the view of a copy's visible columns are named `__mason_…` too, and
         // read as calls, every `SELECT *` over such a copy was cancelled only by a call it never made (a review
@@ -509,7 +746,20 @@ export class SessionRunner {
         const started = Date.now();
         let total = 0;
         let result = null;
-        let closed = false, fetching = false, released = false;
+        let closed = false, fetching = false, released = false, said = false;
+        /** DuckDB ended the query: a fetch failed, or read its end (`cleanUp` says why that is what `release` waits for) */
+        let ended = false;
+        let ending = false;
+        // ITS LINE IS SAID WHEN IT ENDS, before its client hears the end, never when its result is released: that waits for
+        // DuckDB to end it (`cleanUp`), by when its client is gone and the next session's statements are running, and a
+        // reader of the log took the late line for theirs (`tests/duck-wire.test.ts`, 1 run in 100 under load). The
+        // worker's port keeps the order: a line posted before the answer is read before it
+        const say = () => {
+            if (said)
+                return;
+            said = true;
+            this.log(`statement: ${total} row(s) in ${Date.now() - started} ms — ${oneLine(sql)}`);
+        };
         const release = () => {
             if (released)
                 return;
@@ -519,60 +769,32 @@ export class SessionRunner {
             signal.removeEventListener("abort", cancel);
             result?.closeSync();
             p.destroySync();
-            this.log(`statement: ${total} row(s) in ${Date.now() - started} ms — ${oneLine(sql)}`);
+            say();
         };
-        // A CALLING STATEMENT THAT DID NOT RUN TO ITS END (it failed, its cancel raised, it was closed early) IS CLOSED
-        // ONLY ONCE ITS CALLS HAVE STOPPED (the suite hung for good, 0 % CPU). DuckDB hands back a failed chunk
-        // while another of the query's tasks may still wait INSIDE a call — for this thread, which runs every Mason
-        // function — and closing the result waits for that task (Executor::CancelTasks): this thread waited for the task,
-        // the task for this thread, and every later request to the copy with them (`cleanUp`). One that calls no function,
-        // or one read to its end, has no task left to wait for and is released at once
-        const releaseAfterCleanup = () => {
-            if (!callsFunctions || released) {
-                release();
+        // A STATEMENT THAT DID NOT RUN TO ITS END (it failed, its cancel raised, it was closed early) IS CLOSED ONLY ONCE
+        // DuckDB ENDED IT OFF THIS THREAD (`cleanUp`), the session's next statement waiting for that (`releasing`). One read
+        // to its end has no task left, and is released at once
+        const endOffThisThread = () => {
+            if (released || ending)
                 return;
-            }
-            const done = (async () => { await this.cleanUp(seq, sql); release(); })();
+            ending = true;
+            const done = (async () => { await this.cleanUp(seq, sql, callsFunctions, ended ? null : result); release(); })();
             this.releasing.add(done);
             void done.finally(() => this.releasing.delete(done));
         };
-        // a result closed before its end while no fetch is in flight: a statement calling no function is released at
-        // once (the close cancels the executor); one that does is read on until DuckDB hands it back refused (its next
-        // call raises) or read to its end, then released — the session's next statement waits for that (`releasing`)
+        // it ended with an answer its client is about to hear: said now
+        const releaseAfterCleanup = () => { if (!released)
+            say(); endOffThisThread(); };
+        // a result closed before its end while no fetch is in flight: no end its client hears, so its line is its release's
         const drain = () => {
-            const r = result;
             // CLOSED WHILE DuckDB STILL STARTS IT (its `stream()` pending: an aggregate computes there whole): nothing is
             // released under the pending start — that destroyed the prepared statement under it and, its release done, let the
             // start's own failure skip `cleanUp`, so the connection and the instance closed while its other tasks still had
             // calls queued for this thread, and the thread's end met them (Linux CI: `FATAL ERROR: Error::Error
             // napi_define_properties`, 7 runs in 30 of a stop under a calling count). The start's own path releases it: its
             // failure through `releaseAfterCleanup`, its answer drained once handed back (below `stream()`)
-            if (r === null)
-                return;
-            if (!callsFunctions) {
-                release();
-                return;
-            }
-            const done = (async () => {
-                // read on until the next call raises or the result ends — an EMPTY chunk, as `fetch` reads it: past its end a
-                // streamed result hands back empty chunks for ever, never null, and a result closed after its last row but
-                // before its end was read here without end, a thread spinning and the session's close waiting on it
-                try {
-                    for (let c = await r.fetchChunk(); c !== null && c.rowCount > 0; c = await r.fetchChunk()) { /* its rows are nobody's */ }
-                }
-                catch (e) {
-                    // the cancel's own refusal (its next call raised, 57014) is what the drain waits for. Anything else is the
-                    // statement's real error, met after the listener let go of its answer: no one reads it any more, and a
-                    // rethrow here would reach no caller (it would end this thread, and with it every session), so it is said
-                    if (pgErrorOf(e, sql).fields.code !== "57014")
-                        this.log(`a statement closed before its end failed while it was drained: ${e instanceof Error ? e.message : String(e)} — ${oneLine(sql)}`);
-                    // it failed, so a task of it may still be inside a call: closed once its calls have stopped
-                    await this.cleanUp(seq, sql);
-                }
-                release();
-            })();
-            this.releasing.add(done);
-            void done.finally(() => this.releasing.delete(done));
+            if (result !== null)
+                endOffThisThread();
         };
         // an early end by the listener (a portal closed half read, the session gone): cancelled, and released — by the
         // fetch in flight once DuckDB hands the chunk back, else by `drain`
@@ -601,23 +823,23 @@ export class SessionRunner {
             catch (e) {
                 throw failure(e);
             }
-            // CLOSED WHILE IT STARTED (`drain` above), now that DuckDB handed it back. The two differ: one that calls no
-            // function is released at once (its close cancels the executor) and so is never read after — told the cancel
-            // here, before a column of a destroyed result is asked (DuckDB answers 0 columns for one: a review finding, an
-            // XX000 "report it" where the session had closed); one that calls is drained, which holds the result open until
-            // its calls stop, so its answer below is built from a live result and its fetch tells the cancel
+            // CLOSED WHILE IT STARTED (`drain` above), now that DuckDB handed it back, and drained. One that calls no function
+            // is told the cancel here, its release done first, before a column of a destroyed result is asked (DuckDB
+            // answers 0 columns for one: a review finding, an XX000 "report it" where the session had closed); one that calls
+            // holds its result open until DuckDB ended it, so its answer below is built from a live result and its fetch
+            // tells the cancel
             if (closed) {
+                drain();
                 if (!callsFunctions) {
-                    release();
+                    await Promise.all([...this.releasing]);
                     throw cancelled();
                 }
-                drain();
             }
             timing.duckdbMs += performance.now() - t1;
             const r = result;
             if (fields !== null && r.columnCount < fields.length)
-                throw new StatementError({ code: "XX000", message: `DuckDB answered ${r.columnCount} column(s) where Postgres's analysis has ${fields.length}: report it (Mason's rewrite)` }, sql);
-            // Postgres's columns where its analysis gave them (a WITH TIES rank DuckDB adds after them is not sent)
+                throw new StatementError({ code: "XX000", message: `the engine answered ${r.columnCount} column(s) where Postgres's analysis has ${fields.length}: report it (Mason's rewrite)` }, sql);
+            // Postgres's columns where its analysis gave them (the exact numerics' cells DuckDB answers after them are not sent)
             const columns = fields !== null
                 ? fields.map((f, i) => pgTypedColumn(f.name, f.oid, r.columnType(i), f.typmod))
                 : Array.from({ length: r.columnCount }, (_, i) => wireColumn(r.columnName(i), r.columnType(i)));
@@ -646,11 +868,13 @@ export class SessionRunner {
                             chunk = await this.track(r.fetchChunk());
                         }
                         catch (e) {
+                            ended = true;
                             throw failure(e);
                         }
                         finally {
                             fetching = false;
                         }
+                        ended = chunk === null || chunk.rowCount === 0;
                         if (closed) {
                             drain();
                             return null;
@@ -703,8 +927,17 @@ export class SessionRunner {
         return this.track(this.explained(sql, params, session, signal, how));
     }
     async explained(sql, params, session, signal, how) {
-        await Promise.all([...this.releasing]);
         const o = how.options;
+        // A STATEMENT THAT READS A FILTERED TABLE IS NEVER SHOWN TO THE ENGINE: its plan says what it proved from the table's
+        // statistics (docs/DESIGN.md "Roles: rows per user"), so the tables are the statement's and its filters' (a through:'s
+        // parents), and the rest is Mason's plan
+        if ((how.rowFilters ?? []).length > 0) {
+            const tables = [...new Set([...(how.reads ?? []), ...(how.rowFilters ?? []).map((f) => f.table)])].map((t) => { const dot = t.indexOf("."); return { schema: t.slice(0, dot), name: t.slice(dot + 1) }; });
+            const explained = { copies: await this.copies(tables, how.copies, how.rowFilters ?? []), sql: oneLine(sql.text), computed: computedOf(how.exact, how.fields),
+                functions: [...sql.functions], plan: [], filtered: true };
+            return o.format === "json" ? [explainJson(explained, o)] : explainText(explained, o);
+        }
+        await Promise.all([...this.releasing]);
         let plan;
         let analyzed;
         if (!o.analyze)
@@ -727,6 +960,7 @@ export class SessionRunner {
                     }
                 }
                 finally {
+                    await Promise.all([...this.releasing]);
                     await this.conn.run("PRAGMA disable_profiling");
                 }
                 // THE PROFILE IS HELD TO THE STATEMENT, never assumed its: DuckDB writes one when the result is read to its end,
@@ -734,7 +968,7 @@ export class SessionRunner {
                 // whole table is the table's row count), and its plan is its EXPLAIN, the rows the ones it answered
                 const profile = fs.existsSync(file) ? profiled(JSON.parse(fs.readFileSync(file, "utf8"))) : null;
                 if (profile !== null && profile.sql !== sql.text) {
-                    throw new StatementError({ code: "XX000", message: `DuckDB's profile of an EXPLAIN ANALYZE is of another statement (${oneLine(profile.sql ?? "none named")}): report it (Mason's EXPLAIN)` }, sql.text);
+                    throw new StatementError({ code: "XX000", message: `the engine's profile of an EXPLAIN ANALYZE is of another statement (${oneLine(profile.sql ?? "none named")}): report it (Mason's EXPLAIN)` }, sql.text);
                 }
                 if (profile !== null)
                     plan = profile.plan;
@@ -750,7 +984,7 @@ export class SessionRunner {
             }
         }
         // what it calls is the plan's (src/duck/plan.ts PlannedSql), the list that decides how it is cancelled
-        const explained = { copies: await this.copies(scannedTables(plan), how.copies), sql: oneLine(sql.text), computed: computedOf(how.exact, how.fields),
+        const explained = { copies: await this.copies(scannedTables(plan), how.copies, how.rowFilters ?? []), sql: oneLine(sql.text), computed: computedOf(how.exact, how.fields),
             functions: [...sql.functions], plan, ...(analyzed === undefined ? {} : { analyzed }) };
         return o.format === "json" ? [explainJson(explained, o)] : explainText(explained, o);
     }
@@ -788,42 +1022,70 @@ export class SessionRunner {
     }
     /** each copy's rows, and the sync's facts of it (how it is kept current: src/duck/sync.ts `KeptCurrent`, never the
      *  poll's ledger read here — a project switched to logical replication still holds its last poll there) */
-    async copies(tables, facts) {
+    async copies(tables, facts, filters = []) {
         const out = [];
         for (const { schema, name } of tables) {
-            const rows = (await this.conn.runAndReadAll("SELECT estimated_size FROM duckdb_tables() WHERE database_name = current_database() AND schema_name = $1 AND table_name = $2", [schema, name])).getRowsJS()[0]?.[0];
             // the sync's source is the two names as they are; EXPLAIN says the table as a statement writes it (qualifiedName)
-            out.push({ table: qualifiedName({ schema, name }), rows: rows === undefined || rows === null ? null : Number(rows), facts: facts.find((f) => f.source === `${schema}.${name}`) ?? null });
+            const filter = filters.find((f) => f.table === `${schema}.${name}`)?.words;
+            // a table the role filters: its counts are every row's, not the role's, and are never read or said here
+            const rows = filter !== undefined ? null : (await this.conn.runAndReadAll("SELECT estimated_size FROM duckdb_tables() WHERE database_name = current_database() AND schema_name = $1 AND table_name = $2", [schema, name])).getRowsJS()[0]?.[0];
+            out.push({ table: qualifiedName({ schema, name }), rows: rows === undefined || rows === null ? null : Number(rows), facts: facts.find((f) => f.source === `${schema}.${name}`) ?? null,
+                ...(filter !== undefined ? { filter } : {}) });
         }
         return out;
     }
     /**
-     * A CALLING STATEMENT THAT DID NOT RUN TO ITS END, WAITED FOR UNTIL ITS CALLS HAVE STOPPED, before its result is
-     * closed (`releaseAfterCleanup` in `streamed` says why). Every close is synchronous on this thread (DuckDB's node API
-     * has no other, and refuses a second query on a connection with a result open), so the close must find no task of the
-     * query inside, or waiting for, a call. The statement is marked cancelled (each call it still makes raises at once)
-     * and interrupted (its tasks stop at their next chunk), and this thread stays free — a task waiting for a call is
-     * answered within a turn of its loop — until no call of this session has been answered for `CALLS_QUIET_MS`. Never a
-     * deadline: a close while calls still come is the deadlock itself, and the calls end (each raises); a wait past
-     * `CALLS_SLOW_MS` is said once.
+     * A STATEMENT THAT DID NOT RUN TO ITS END, ENDED BY DuckDB OFF THIS THREAD before its result is closed. THIS THREAD
+     * NEVER WAITS FOR THE ENGINE'S TASKS: every close is synchronous here, a close of an open result waits for the query's
+     * tasks (Executor::CancelTasks), and this thread runs every Mason function, so a task inside a call waits for it (the
+     * suite hung for good at 0 % CPU; `tests/duck-executor.test.ts` "a vector slower than"). Waiting for quiet instead —
+     * no call for 50 ms, as this did — was no proof: a vector whose native work outlasts the window has made no call yet.
+     * And a call-free statement's tasks hold this thread as long as DuckDB's vector takes, with every session behind it.
+     *
+     * WHAT WAITS IS A FETCH (read in DuckDB 2.0's source, `ClientContext::EndQueryInternal`, and held by the test above).
+     * Every end of a query runs `CancelTasks` on the thread that ends it, and a fetch runs on libuv's pool. Interrupted
+     * first, the next fetch does not compute: it fails at once (`BufferedData::Participate`), ending the query there, its
+     * tasks waited for while this thread answers their calls; or it reads the end, or the end was read already (`r`
+     * null). So the wait is bounded by DuckDB's next chunk, on a statement that never ends by itself too, and an EMPTY chunk
+     * is the end, as `fetch` reads it: past its end a streamed result hands back empty chunks for ever, never null.
+     *
+     * THE PROBE PROVES IT (a statement performs no wait: while a result holds the connection DuckDB refuses it, "connection
+     * has an open result", before anything else). So a `SELECT 1` that ran is proof that nothing of the statement is left,
+     * and the close finds nothing to wait for. Refused, the reading above no longer holds: said as a Mason bug, and closed
+     * all the same, the watch (src/duck/executor.ts SILENT_MS) catching the deadlock if there is one. A wait past
+     * `ENDING_SLOW_MS` is said once.
      */
-    async cleanUp(seq, sql) {
-        cancelStatement(this.cancels, this.slot, seq);
+    async cleanUp(seq, sql, calls, r) {
+        if (calls)
+            cancelStatement(this.cancels, this.slot, seq);
         this.conn.interrupt();
-        const t0 = performance.now();
-        let said = false;
-        // ONE WHOLE WINDOW BEFORE THE FIRST LOOK: a call a task has queued for this thread but that has not run yet leaves
-        // `msSinceCall` at the previous call's time — long ago, when this thread was busy with another connection's close
-        // (an interrupt honoured, ~10 s for one join) — so a check made synchronously here could pass with that call still
-        // waiting, and the close deadlock on it the same way. A sleep of the window lets every queued call run first.
-        for (;;) {
-            await sleep(CALLS_QUIET_MS);
-            if (msSinceCall(this.slot) >= CALLS_QUIET_MS)
-                break;
-            if (!said && performance.now() - t0 > CALLS_SLOW_MS) {
-                said = true;
-                this.log(`a statement that did not run to its end still calls Mason's functions after ${CALLS_SLOW_MS / 1000} s; its result is closed once they stop — ${oneLine(sql)}`);
+        const slow = setTimeout(() => this.log(`a statement that did not run to its end has not stopped after ${ENDING_SLOW_MS / 1000} s; its result is closed once the engine ended it — ${oneLine(sql)}`), ENDING_SLOW_MS);
+        try {
+            if (r !== null) {
+                try {
+                    for (let c = await r.fetchChunk(); c !== null && c.rowCount > 0; c = await r.fetchChunk()) { /* its rows are nobody's */ }
+                }
+                catch (e) {
+                    // the cancel's own refusal (the interrupt, or its next call raised, 57014) is what this waits for. Anything else
+                    // is the statement's real error, met after the listener let go of its answer: no one reads it any more, and a
+                    // rethrow here would reach no caller (it would end this thread, and with it every session), so it is said
+                    if (pgErrorOf(e, sql).fields.code !== "57014")
+                        this.log(`a statement closed before its end failed while it was drained: ${e instanceof Error ? e.message : String(e)} — ${oneLine(sql)}`);
+                }
+            }
+            // an abort landing now interrupts the probe past DuckDB's check, which is its proof all the same
+            try {
+                await this.conn.run("SELECT 1");
             }
+            catch (e) {
+                if (/open result/.test(String(e)))
+                    this.log(`a statement's result is still open after the engine failed or ended it (Mason bug, report it with mason logs): closed all the same — ${oneLine(sql)}`);
+                else if (pgErrorOf(e, "SELECT 1").fields.code !== "57014")
+                    this.log(`the check that a statement ended failed: ${e instanceof Error ? e.message : String(e)} — ${oneLine(sql)}`);
+            }
+        }
+        finally {
+            clearTimeout(slow);
         }
     }
     /** Closes the connection once nothing runs on it: an answer left open drained (its statement cancelled), every
@@ -858,7 +1120,7 @@ function exactColumns(columns, fields, exact, typeAt) {
     const computed = exact.filter((e) => {
         const unreadable = readers.some((r, j) => r === null && programReads(e, j));
         if (unreadable)
-            columns[e.column] = { ...columns[e.column], divergence: `Postgres computes this numeric exactly; a part of it DuckDB computed as a double` };
+            columns[e.column] = { ...columns[e.column], divergence: `Postgres computes this numeric exactly; a part of it the engine computed as a double` };
         return !unreadable;
     });
     if (computed.length === 0)
@@ -882,7 +1144,7 @@ function exactRows(computed, rows, base, sql, params) {
             }
             catch (err) {
                 if (err instanceof NumericError)
-                    throw new StatementError({ code: err.sqlstate, message: err.message }, sql);
+                    throw new StatementError({ code: err.sqlstate, message: err.message, ...(err.detail === null ? {} : { detail: err.detail }) }, sql, err.gap);
                 throw err;
             }
         }
@@ -893,11 +1155,11 @@ function exactRows(computed, rows, base, sql, params) {
 function computedOf(exact, fields) {
     return exact.map((e) => {
         const cells = new Set(cellsOf(e.program).filter((k) => !k.startsWith("$"))).size;
-        return `column ${e.column + 1}${fields?.[e.column] === undefined ? "" : ` "${fields[e.column].name}"`}: Postgres's exact numeric${e.text ? " as text" : ""}, from ${cells} of DuckDB's value(s)`;
+        return `column ${e.column + 1}${fields?.[e.column] === undefined ? "" : ` "${fields[e.column].name}"`}: Postgres's exact numeric${e.text ? " as text" : ""}, from ${cells} of the engine's value(s)`;
     });
 }
 /** whether the column's program reads cell j */
-const programReads = (e, j) => JSON.stringify(e.program).includes(`"key":"${j}"`);
+const programReads = (e, j) => cellsOf(e.program).includes(String(j));
 /** a statement on one line: whole in the log (its ORDER BY, past 160 characters, was what told a slow preview's
  *  cause), cut where it goes to a client in an error's detail */
 const oneLine = (sql, max = Infinity) => { const t = sql.replace(/\s+/g, " ").trim(); return t.length > max ? `${t.slice(0, max - 3)}…` : t; };