@mason-data/cli 0.8.0 → 0.10.0

package/dist/src/duck/store.js

@@ -19,16 +19,16 @@ import * as os from "node:os";
 import { DuckDBInstance, version } from "@duckdb/node-api";
 import { Executor } from "./executor.js";
 import { beginStatement, cancelStatement, slotName, statementCancelled } from "./cancel.js";
-import { StatementError } from "../api/postgres/server.js";
-import { duckParameters, missingRelationHint, planDuck, quoteDuckKeywords } from "./plan.js";
+import { StatementError } from "./statement.js";
+import { duckParameters, missingRelationHint, ordersText, planDuck, quoteDuckKeywords } from "./plan.js";
 import { VERIFIED } from "./verified.js";
 import { registerFunctions, SESSION_VARIABLE, SETTINGS_VARIABLE } from "./functions.js";
-import { duckCollation } from "./rewrite.js";
-import { decodeLibcOrder, encodeLibcOrder } from "./libc-order.js";
+import { caseMappingOf, CollationResolver, duckCollation, isLibcCollation } from "./collation.js";
+import { decodeLibcOrder, encodeLibcOrder, isCurrentLibcOrder } from "./libc-order.js";
 import { dataRow } from "../api/postgres/protocol.js";
 import { cellText, exactColumn, pgTypedColumn, wireColumn } from "./types.js";
 import { gapIn, gapMark } from "./gaps.js";
-import { cellsOf, evaluateText, NumericError } from "../lib/pg-numeric.js";
+import { cellsOf, evaluateText, fieldOverflow, NumericError } from "../lib/pg-numeric.js";
 import { explainJson, explainText, planNodes, profiled, qualifiedName, scannedTables } from "./explain.js";
 import * as fs from "node:fs";
 import * as path from "node:path";
@@ -65,20 +65,94 @@ const INSTANCE_SETTINGS = { late_materialization_max_rows: "10000", force_compre
 /**
  * DUCKDB'S MEMORY, BOUNDED FOR THIS PROCESS: DuckDB's default is 80% of the machine's RAM, in a process
  * that also holds V8's heap and two PGlites, which ends with the system killing every session and the sync together.
- * Half the RAM unless `MASON_DUCKDB_MEMORY` says (`8GB`, `512MiB`); past it a sort, a join or an aggregate spills to
- * `<file>.tmp` (DuckDB's own default, named here so it is the copy's folder), and what cannot spill fails ITS statement
- * with 53200 and the way out (pgErrorOf), the run going on.
+ * Unless `MASON_DUCKDB_MEMORY` says (`8GB`, `512MiB`), half the memory, or what leaves `OUTSIDE_RESERVE_BYTES` beside
+ * it when that is less (`defaultMemoryLimit`); past it a sort, a join or an aggregate spills to `<file>.tmp` (DuckDB's
+ * own default, named here so it is the copy's folder), and what cannot spill fails ITS statement with 53200 and the way
+ * out (pgErrorOf), the run going on.
+ *
+ * THE ROOM OUTSIDE THE LIMIT (issue 301, docs/DESIGN.md "Memory: the room outside the engine's limit"): the limit counts
+ * DuckDB's buffers, never Node's heap, the catalog's PGlite, the pg-eval PGlite nor what DuckDB allocates outside its
+ * buffer manager, and with half of a 1, 1.5, 2 or 3 GiB container that room was too small: the kernel killed the run in
+ * each. Measured with the limit at 256MiB, the process reached 2.3 GiB beside it (the pg-eval PGlite alone ~0.5 GiB of
+ * WebAssembly memory once a statement handed it 1.5 M values, the catalog's ~0.26 GiB). `OUTSIDE_RESERVE_BYTES` is that
+ * peak with a margin; `MEMORY_FLOOR_BYTES` the smallest container the whole path (a first sync, a generated table's
+ * build, writes on the stream, statements through the listener) ran in under this default. Under the floor, with
+ * `MASON_DUCKDB_MEMORY` unset, a start goes on and WARNS (`memoryWarning`: the run's start, mason status, mason doctor),
+ * its statements given MIN_SHARE_BYTES, the least room for them and the most for the rest (Tom's decision on PR 382: a
+ * warning, never a refusal); set, the variable is the operator's word at any size.
+ */
+export const OUTSIDE_RESERVE_BYTES = 2560 * 2 ** 20;
+export const MEMORY_FLOOR_BYTES = 3072 * 2 ** 20;
+/** the statements' share where the reserve leaves less (under the floor, where a start warns): what let the run last
+ *  longest, measured (docs/DESIGN.md): at 2 GiB the whole path ran with it, as with 256MiB, and at 1.5 GiB it answered
+ *  every statement before the kill where 256MiB was killed in the writes; the same statements answered at both, one
+ *  wide aggregate failing alone (53200), and the old quarter, 512MiB, was killed at 2 GiB */
+export const MIN_SHARE_BYTES = 128 * 2 ** 20;
+/**
+ * The default memory limit for `totalBytes` of memory, in whole MiB (mason doctor says it for any machine): half of it,
+ * the page cache keeping the other half, or what leaves OUTSIDE_RESERVE_BYTES beside it when that is less (from 5 GiB
+ * up it is half), never under MIN_SHARE_BYTES (from 2.625 GiB down: under the floor, where a start warns), nor past half.
+ */
+export function defaultMemoryLimit(totalBytes) {
+    const mib = 2 ** 20;
+    const half = Math.floor(totalBytes / 2 / mib), room = Math.floor((totalBytes - OUTSIDE_RESERVE_BYTES) / mib);
+    return `${Math.min(half, Math.max(MIN_SHARE_BYTES / mib, room))}MiB`;
+}
+/** A size as a reader recognises it: GiB to one decimal from 1 GiB up, else MiB. */
+export function memorySize(bytes) {
+    return bytes >= 2 ** 30 ? `${(bytes / 2 ** 30).toFixed(1)} GiB` : `${Math.round(bytes / 2 ** 20)} MiB`;
+}
+/** `MASON_DUCKDB_MEMORY` as it is set: null unset (or set empty, as the run reads it) */
+function memorySet(env) {
+    const v = env.MASON_DUCKDB_MEMORY;
+    return v === undefined || v === "" ? null : v;
+}
+/** `MASON_DUCKDB_MEMORY` that is not an amount: the one text, for the store's open, mason serve, mason start and doctor */
+export function memorySettingWrong(text) {
+    return { message: `MASON_DUCKDB_MEMORY=${JSON.stringify(text)} is not an amount of memory`,
+        why: "it is the memory the synced data's statements may use before they spill to disk",
+        fix: `write it as 8GB or 512MiB, or unset it for the default (${DEFAULT_MEMORY_LIMIT} here: ${DEFAULT_MEMORY_RULE})` };
+}
+/** THE DEFAULT'S RULE IN WORDS, the one text every message, help and guide that states the default says (a fact derived
+ *  in several places: PR 380's first-sync refusal said "half the machine's RAM" after the default had changed);
+ *  `defaultMemoryLimit` is the rule, and `tests/conventions.test.ts` holds src/ to saying it only through this */
+export const DEFAULT_MEMORY_RULE = `half the container's memory limit, else of the machine's RAM, or what leaves ${memorySize(OUTSIDE_RESERVE_BYTES)} ` +
+    `beside it when that is less, never under ${memorySize(MIN_SHARE_BYTES)}`;
+/**
+ * WHAT A START REFUSES OF THE MEMORY (mason serve's one report; mason start and mason doctor take the first):
+ * `MASON_DUCKDB_MEMORY` that is not an amount. A size under the floor is never refused: it is `memoryWarning`'s.
  */
-/** The default memory limit for a machine with `totalBytes` of RAM: half of it, in MiB (mason doctor says it for any machine). */
-export function defaultMemoryLimit(totalBytes) { return `${Math.floor(totalBytes / 2 / 2 ** 20)}MiB`; }
+export function memoryProblems(env) {
+    const set = memorySet(env);
+    return set !== null && !MEMORY_LIMIT_RE.test(set) ? [memorySettingWrong(set)] : [];
+}
+/**
+ * THE ONE WARNING OF THE MEMORY: `memory` under MEMORY_FLOOR_BYTES while `MASON_DUCKDB_MEMORY` is unset (or not an
+ * amount, which is refused beside it). The start goes on; the run says it at its start and in what mason status reads
+ * (its warnings), and mason doctor's memory line says it, each from this text alone. `memory` is `machineMemory()`.
+ */
+export function memoryWarning(memory, env) {
+    const set = memorySet(env);
+    if ((set !== null && MEMORY_LIMIT_RE.test(set)) || memory.bytes >= MEMORY_FLOOR_BYTES)
+        return null;
+    const size = memorySize(memory.bytes);
+    return { message: `${size} of memory is under the ${memorySize(MEMORY_FLOOR_BYTES)} Mason needs`,
+        why: `${size} is ${memory.from === "container" ? "the container's memory limit" : "the machine's RAM"}; ` +
+            `beside what the synced data's statements may use, Mason itself needs up to ${memorySize(OUTSIDE_RESERVE_BYTES)} (measured, with a margin), ` +
+            "and with less the system may stop the whole run, every session with it, when a first sync, a burst of writes or a statement needs the memory",
+        fix: `give it ${memorySize(MEMORY_FLOOR_BYTES)} or more (the container's memory limit), or set MASON_DUCKDB_MEMORY=<n> ` +
+            "to choose the statements' share yourself, the rest left to Mason (mason doctor says what it leaves)" };
+}
+/** a problem or the warning as one line: mason serve's settings report, the run's warning, mason doctor's detail */
+export function memoryProblemText(p) { return `${p.message}: ${p.why} — ${p.fix}`; }
 /**
  * THE MEMORY THIS PROCESS MAY USE: the machine's RAM, or the container's limit where one is set — a cgroup v2
  * `memory.max` or a cgroup v1 `memory.limit_in_bytes` below it. `os.totalmem()` is the HOST's RAM inside a container,
  * so half of it was more than a limited container had, and the kernel killed the run instead of DuckDB spilling
  * (the review of the cut). `read` is the test's.
  */
-export function machineMemoryBytes(read = (f) => fs.readFileSync(f, "utf8"), total = os.totalmem()) {
-    let limit = total;
+export function machineMemory(read = (f) => fs.readFileSync(f, "utf8"), total = os.totalmem()) {
+    let limit = { bytes: total, from: "machine" };
     for (const file of ["/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"]) {
         let text;
         try {
@@ -89,12 +163,35 @@ export function machineMemoryBytes(read = (f) => fs.readFileSync(f, "utf8"), tot
         } // not a cgroup of that version, or not a container: the machine's RAM
         // v2 says `max` for no limit; v1 says a number near 2^63; either way a number below the RAM is the limit
         const n = Number(text);
-        if (text !== "max" && Number.isFinite(n) && n > 0 && n < limit)
-            limit = n;
+        if (text !== "max" && Number.isFinite(n) && n > 0 && n < limit.bytes)
+            limit = { bytes: n, from: "container" };
     }
     return limit;
 }
+/** `machineMemory`'s bytes: the one size the default limit and the floor are read against */
+export function machineMemoryBytes(read, total) { return machineMemory(read, total).bytes; }
 export const DEFAULT_MEMORY_LIMIT = defaultMemoryLimit(machineMemoryBytes());
+/**
+ * WHAT STATEMENTS MAY SPILL (issue 105: a 100-row preview's sort took every free byte of a volume, and the sync's room
+ * with it). DuckDB's default ceiling is "90% of available disk space", and what it answered when the disk ran out first
+ * was `XX000` with a path. Here the ceiling is HALF of the room statements have, what is free where the copy is and what
+ * they have spilled there (the disk's free bytes exclude the spill, so a ceiling of the free bytes alone would grow with
+ * each read as the spill grows, and end near the whole disk). The sync's writes keep the other half whatever statements
+ * do, and it follows the disk (`DuckStore.keepSpillCeiling`, every `SPILL_REFRESH_MS`): a copy that grows narrows it.
+ * `MASON_SPILL_LIMIT` sets the amount instead, never past 90% of that room. Spilled files are shared by the statements
+ * running, so two at once share the ceiling. Past it the statement that needed the room fails alone, 53400 as past
+ * Postgres's temp_file_limit, with the way out (pgErrorOf), and its files are gone with it; a disk that is really full
+ * is 53100. A sizing decision reads `store.spillLimitBytes`, as it reads `store.memoryLimitBytes`.
+ */
+const SPILL_FREE_SHARE = 0.5, SPILL_FREE_MOST = 0.9;
+const SPILL_REFRESH_MS = 15_000;
+export function spillCeiling(freeBytes, spilledBytes, setBytes) {
+    const room = freeBytes + spilledBytes;
+    return Math.max(2 ** 20, Math.floor(setBytes === null ? room * SPILL_FREE_SHARE : Math.min(setBytes, room * SPILL_FREE_MOST)));
+}
+/** the bytes free where `dir` is: the one reading of it (the sync's disk check, the spill's ceiling) */
+export function freeBytesAt(dir) { const st = fs.statfsSync(dir); return st.bavail * st.bsize; }
+const spillSetting = (bytes) => `${Math.floor(bytes / 2 ** 20)}MiB`;
 export const MEMORY_LIMIT_RE = /^\s*(\d+(?:\.\d+)?)\s*(KB|MB|GB|TB|KiB|MiB|GiB|TiB)\s*$/i;
 /** DuckDB's memory limit in bytes, as DuckDB reads the text: KB / MB / GB / TB are powers of 1000, KiB / MiB / GiB / TiB of 1024 */
 export function memoryLimitBytes(text) {
@@ -139,17 +236,31 @@ export class DuckStore {
     file;
     /** the memory limit every connection shares, in bytes */
     memoryLimitBytes;
+    /** what statements may spill now, in bytes (`spillCeiling`): it follows the disk */
+    spillLimitBytes;
+    spillSet;
+    diskFree;
+    spillTimer;
+    spillLane = null;
+    spillKept = null;
+    log;
     executor;
     closed = null;
     closeGraceMs;
     /** where the copy's extensions come from: Mason's package for this platform, or DuckDB's install (src/duck/extensions.ts) */
     extensions;
-    constructor(file, executor, memoryLimit, extensions, closeGraceMs) {
+    constructor(file, executor, memoryLimit, extensions, spill, log, closeGraceMs) {
         this.file = file;
         this.executor = executor;
         this.memoryLimitBytes = memoryLimit;
         this.extensions = extensions;
         this.closeGraceMs = closeGraceMs;
+        this.spillLimitBytes = spill.limit;
+        this.spillSet = spill.set;
+        this.diskFree = spill.diskFree;
+        this.log = log;
+        this.spillTimer = setInterval(() => { void this.keepSpillCeiling(); }, SPILL_REFRESH_MS);
+        this.spillTimer.unref();
         this.writer = executor.writer();
         this.instance = { connect: () => executor.connect() };
     }
@@ -160,8 +271,13 @@ export class DuckStore {
         const extensions = options.extensions ?? findExtensions();
         const memoryLimit = options.memoryLimit ?? DEFAULT_MEMORY_LIMIT;
         if (!MEMORY_LIMIT_RE.test(memoryLimit))
-            throw new Error(`MASON_DUCKDB_MEMORY=${JSON.stringify(memoryLimit)} is not an amount of memory: write it as 8GB or 512MiB (unset, half the machine's RAM: ${DEFAULT_MEMORY_LIMIT})`);
+            throw new Error(memoryProblemText(memorySettingWrong(memoryLimit)));
+        if (options.spillLimit !== undefined && !MEMORY_LIMIT_RE.test(options.spillLimit))
+            throw new Error(`MASON_SPILL_LIMIT=${JSON.stringify(options.spillLimit)} is not an amount of disk: write it as 40GB or 512MiB (unset, half of what is free on the disk)`);
+        const spillSet = options.spillLimit === undefined ? null : memoryLimitBytes(options.spillLimit);
+        const diskFree = options.diskFree ?? freeBytesAt;
         const where = path.resolve(file);
+        const spillLimit = spillCeiling(diskFree(path.dirname(where)), 0, spillSet);
         const before = opened.get(where);
         if (before !== undefined && before.closed === null)
             throw new Error(`Mason could not open the synced data ${file}: this process has it open already (Mason bug: report it)`);
@@ -175,7 +291,7 @@ export class DuckStore {
             if (before !== undefined)
                 await before.closed;
             // every connection's, the sync's and each session's
-            executor = await Executor.start({ file, settings: { ...INSTANCE_SETTINGS, ...extensionSettings(extensions), memory_limit: memoryLimit, temp_directory: `${file}.tmp` }, log,
+            executor = await Executor.start({ file, settings: { ...INSTANCE_SETTINGS, ...extensionSettings(extensions), memory_limit: memoryLimit, temp_directory: `${file}.tmp`, max_temp_directory_size: spillSetting(spillLimit) }, log,
                 ...(options.silentMs !== undefined ? { silentMs: options.silentMs } : {}), ...(options.workerFile !== undefined ? { workerFile: options.workerFile } : {}) });
         }
         catch (e) {
@@ -183,7 +299,7 @@ export class DuckStore {
                 opened.delete(where);
             throw e;
         }
-        const store = new DuckStore(file, executor, memoryLimitBytes(memoryLimit), extensions, options.closeGraceMs);
+        const store = new DuckStore(file, executor, memoryLimitBytes(memoryLimit), extensions, { limit: spillLimit, set: spillSet, diskFree }, log, options.closeGraceMs);
         // closed: the file may be opened again — unless its thread was given up on (`abandoned`), which keeps it here
         store.closing = (p) => { mine.closed = p; void p.then(() => { if (executor.abandoned !== null)
             mine.abandoned = executor.abandoned;
@@ -192,6 +308,45 @@ export class DuckStore {
         return store;
     }
     closing = () => undefined;
+    /**
+     * THE SPILL'S CEILING MOVED WITH THE DISK (`spillCeiling`): the bytes free and spilled now, set on the instance on a
+     * connection of its own, never the writer's. One at a time; a move under 1 % is none. DuckDB refuses a ceiling under
+     * what is spilled at that moment (more may have spilled since the read): it stays where it was until the next turn.
+     * Any other failure is said; the ceiling stands, which is the last one set.
+     */
+    keepSpillCeiling() {
+        if (this.closed !== null)
+            return Promise.resolve();
+        this.spillKept ??= this.spillTurn().catch((e) => {
+            if (this.closed === null)
+                this.log(`the spill's ceiling was not moved: ${e instanceof Error ? e.message : String(e)}`);
+        }).finally(() => { this.spillKept = null; });
+        return this.spillKept;
+    }
+    async spillTurn() {
+        let c;
+        // a connection that failed is asked for again next turn, never kept as the answer
+        try {
+            c = await (this.spillLane ??= this.executor.connect());
+        }
+        catch (e) {
+            this.spillLane = null;
+            throw e;
+        }
+        const spilled = Number((await c.runAndReadAll("SELECT coalesce(sum(size), 0) FROM duckdb_temporary_files()")).getRowsJS()[0][0]);
+        const next = spillCeiling(this.diskFree(path.dirname(path.resolve(this.file))), spilled, this.spillSet);
+        if (Math.abs(next - this.spillLimitBytes) < this.spillLimitBytes * 0.01)
+            return;
+        try {
+            await c.run(`SET GLOBAL max_temp_directory_size = '${spillSetting(next)}'`);
+        }
+        catch (e) {
+            if (e instanceof Error && e.message.includes("failed to adjust"))
+                return;
+            throw e;
+        }
+        this.spillLimitBytes = next;
+    }
     /** called once if the engine's thread dies while the run uses it (src/duck/executor.ts): the run exits with the reason */
     onCrash(f) { this.executor.onCrash(f); }
     /** how many vectors each Mason function was called with so far, on the engine's thread (a test's, a measurement's) */
@@ -202,13 +357,14 @@ export class DuckStore {
     runner() { return this.executor.runner(); }
     /**
      * The source's order of text under a libc collation (src/duck/libc-order.ts), kept in the copy: the one recorded
-     * for this collation and version, else `read` from the source (a few seconds) and recorded. `read: true` when it was.
+     * for this collation and version, in this release's form, else `read` from the source (a few seconds) and recorded.
+     * `read: true` when it was.
      */
     async libcOrder(collate, version, read) {
         await this.writer.run(`CREATE SCHEMA IF NOT EXISTS ${MASON_SCHEMA}`);
         await this.writer.run(`CREATE TABLE IF NOT EXISTS ${MASON_SCHEMA}.libc_order (collation_name VARCHAR, collation_version VARCHAR, data VARCHAR)`);
         const kept = (await this.writer.runAndReadAll(`SELECT data FROM ${MASON_SCHEMA}.libc_order WHERE collation_name = $1 AND collation_version IS NOT DISTINCT FROM $2`, [collate, version])).getRowsJS();
-        if (kept.length > 0)
+        if (kept.length > 0 && isCurrentLibcOrder(String(kept[0][0])))
             return { order: decodeLibcOrder(String(kept[0][0])), read: false };
         const order = await read();
         // one order per collation: a version that moved replaces the last one's
@@ -220,22 +376,50 @@ export class DuckStore {
      *  its text sorted as the SOURCE sorts it (`collation`, pg_database's; absent, by bytes; a libc one by `libc`, its
      *  order as `libcOrder` read it), refusing what is not `verified` (null: nothing is refused for it, the oracle's
      *  recording run alone). */
-    async listener(collation, verified = VERIFIED, libc, copies = () => [], kept) {
+    async listener(collation, verified = VERIFIED, libc, copies = () => [], kept, more = {}) {
         const known = new Set((await this.writer.runAndReadAll("SELECT collname FROM pragma_collations()")).getRowsJson().map((r) => String(r[0])));
         quoteDuckKeywords((await this.writer.runAndReadAll("SELECT keyword_name FROM duckdb_keywords() WHERE keyword_category <> 'unreserved'")).getRowsJson().map((r) => String(r[0])));
-        const c = collation === undefined ? { kind: "bytes" } : duckCollation(collation.provider, collation.collate, collation.locale, known);
+        const named = collation === undefined ? { kind: "bytes" } : duckCollation(collation.provider, collation.collate, collation.locale, known);
+        // (a libc locale whose strcoll is byte order, musl's or macOS's: src/duck/libc-order.ts)
+        const c = named.kind === "libc" && libc?.bytes === true ? { kind: "bytes" } : named;
         if (c.kind === "libc") {
             if (libc === undefined)
                 throw new Error(`the database orders text by the libc collation ${collation.collate}, and its order was not read (Mason bug: report it)`);
-            await this.executor.registerLibcOrder(libc);
+            await this.executor.registerLibcOrder(c.collate, libc, true);
         }
+        // each other libc collation read at the start (a synced column's): its key's order, by its collcollate
+        for (const [collate, order] of more.libcOrders ?? [])
+            if (!order.bytes)
+                await this.executor.registerLibcOrder(collate, order, false);
+        // a libc collation named first by a statement: read from the source on its first use, kept in the copy, then the
+        // engine told its order before the plan that needs it is made (src/duck/collation.ts)
+        const readLibc = more.readLibc;
+        // the database's case mapping is its CTYPE's (datctype; ICU's or builtin's locale), never its order's
+        const ctype = collation === undefined ? null : collation.provider === "c" ? collation.ctype ?? collation.collate : collation.locale ?? collation.collate;
+        const resolver = new CollationResolver({ database: c, databaseCollate: collation !== undefined && isLibcCollation(collation.provider, collation.collate) ? collation.collate : null,
+            ...(collation === undefined ? {} : { databaseCase: { mapping: caseMappingOf(collation.provider, ctype, collation.locale), ctype: ctype ?? "" } }),
+            source: more.collations ?? null, known, libc: more.libcOrders ?? new Map(),
+            readLibc: readLibc === undefined ? null : async (sc) => {
+                const r = await this.libcOrder(sc.collate ?? "C", sc.version ?? null, () => readLibc(sc));
+                if (!r.order.bytes)
+                    await this.executor.registerLibcOrder(sc.collate ?? "C", r.order, false);
+                return r.order;
+            } });
+        // the database's own order, held where a plan reads each order by its collcollate (src/duck/libc-bound.ts)
+        if (c.kind === "libc" && libc !== undefined)
+            resolver.hold(c.collate, libc);
         // a jsonb's stored key is this collation's (src/duck/kept.ts `orderBy`)
         await kept?.orderBy(c, c.kind === "libc" ? libc : undefined);
         return {
             collation: c,
+            collations: resolver,
             open: () => this.runner(),
-            plan: (catalog, text, stmt, base, paramOids, searchPath, timeZone, userTypes, wireBound, grants) => planDuck(catalog, text, stmt, base, paramOids, searchPath, timeZone, c, verified, userTypes, copies, 0, kept ?? null, wireBound ?? false, grants),
+            plan: (catalog, text, stmt, base, paramOids, under, userTypes, wireBound, grants, relations, outputs) => planDuck({ catalog, text, stmt, base, paramOids, searchPath: under.searchPath, timeZone: under.timeZone, dateStyle: under.dateStyle, collations: resolver, verified, userTypes, copies, kept: kept ?? null, wireBound: wireBound ?? false, grants, ...(relations === undefined ? {} : { relations }), outputs: outputs ?? false }),
             parameters: duckParameters,
+            // a SELECT of no table that orders text under a collation that is not byte order: the synced data's (issue 122);
+            // one that maps case (upper, lower, initcap: issue 406) is the synced data's where its run answers it (the
+            // listener's `answersCase`), else the catalog's, as before
+            ordersText: (catalog, text, paramOids, under, bindable) => ordersText({ catalog, text, paramOids, under, collations: resolver, bindable }),
             // the sync's state, which EXPLAIN says how each copy is kept current from (src/duck/sync.ts KeptCurrent)
             copies,
             // a stored JSON path ready or gone (src/duck/kept.ts): the plans kept are made again
@@ -246,6 +430,7 @@ export class DuckStore {
      *  (a second open of the file waits for it), or once the executor's grace has passed whatever the thread does
      *  (src/duck/executor.ts CLOSE_GRACE_MS); never rejects — a failure is said in the log. */
     close() {
+        clearInterval(this.spillTimer);
         if (this.closed === null) {
             this.closed = this.executor.close(this.closeGraceMs);
             this.closing(this.closed);
@@ -281,11 +466,22 @@ export function pgErrorOf(err, sql) {
     const full = err instanceof Error ? err.message : String(err);
     const kind = /^([A-Za-z ]+) Error: /.exec(full)?.[1] ?? "";
     const message = full.replace(/^[A-Za-z ]+ Error: /, "").replace(/\n\n?LINE \d+:[\s\S]*$/, "").trim();
+    // NO ROOM TO SPILL (issue 105): the ceiling, or the disk itself before it. Postgres's class for a full disk, in
+    // Mason's words: the engine's text names a path and a setting no client of Mason has, so it stays out of the answer
+    // Read by the engine's own kind and sentence: a statement's literal may hold the same words in any other error
+    const ceiling = kind === "Out of Memory" && /This limit was set by the 'max_temp_directory_size' setting/.test(full);
+    if (ceiling || (kind === "IO" && /No space left on device/.test(full))) {
+        return new StatementError({ code: ceiling ? "53400" : "53100", message: ceiling ? "could not write to temporary file: this statement needs more room on disk than statements may take"
+                : "could not write to temporary file: No space left on device",
+            hint: "ask for less at once (a filter, a LIMIT, fewer or narrower columns, a coarser GROUP BY), free disk where the synced data is, or raise MASON_SPILL_LIMIT (40GB; by default half of what is free) and start mason again; nothing else stopped" }, sql, "spill-limit");
+    }
     const pg = postgresWording(kind, message);
     // a refusal raised inside DuckDB (a Mason function's, the rewrite's error() call) crossed it as its text alone: the
     // entry its text names (gapIn), read here, the one place that text becomes a statement's error
+    // (Mason's own refusal keeps the engine's text out of the DETAIL: it may quote the row, tests/conventions.test.ts)
+    // (a Mason function's error that carries Postgres's own DETAIL sends it as Postgres does, apart from the message)
     if (pg !== null)
-        return new StatementError({ code: pg.code, message: pg.message, detail: `the engine: ${message}` }, sql, pg.gap ?? (pg.code === "0A000" ? gapIn(pg.message) : null));
+        return new StatementError({ code: pg.code, message: pg.message, ...(pg.code === "0A000" ? {} : { detail: pg.detail ?? `the engine: ${message}` }) }, sql, pg.gap ?? (pg.code === "0A000" ? gapIn(pg.message) : null));
     const fn = /(?:Scalar|Aggregate|Table) Function with name "?([A-Za-z_][\w]*)"? does not exist/.exec(message)?.[1];
     if (fn !== undefined) {
         // what reaches DuckDB was verified against Postgres (src/duck/verified.ts), so a function DuckDB lacks is a gap in
@@ -314,9 +510,12 @@ export function pgErrorOf(err, sql) {
                                     : kind === "Out of Range" ? "22003"
                                         : kind === "Interrupt" || /INTERRUPT/i.test(full) ? "57014"
                                             : kind === "Out of Memory" ? "53200"
-                                                : kind === "Not implemented" ? "0A000"
-                                                    : kind === "Invalid Input" ? "22023"
-                                                        : "XX000";
+                                                // a commit that could not pin or allocate a block is the same limit reached, under the transaction's own kind
+                                                // (measured on DuckDB 2.0.0: "Failed to commit: failed to pin block of size 1.0 MiB (365.7 MiB/366.2 MiB used)")
+                                                : kind === "TransactionContext" && /^Failed to commit: failed to (?:pin block|allocate data) of size .* used\)/.test(message) ? "53200"
+                                                    : kind === "Not implemented" ? "0A000"
+                                                        : kind === "Invalid Input" ? "22023"
+                                                            : "XX000";
     return new StatementError({ code, message: code === "57014" ? "canceling statement due to user request"
             : code === "53200" ? "out of memory: this statement needs more than the memory Mason's engine may use, in a part that cannot spill to disk" : message,
         // DuckDB's own advice is SET, which a client of Mason cannot do: Mason's way out instead
@@ -345,16 +544,23 @@ function postgresWording(kind, message) {
     const field = /^Date out of range: (.*)$/.exec(message);
     if (field)
         return { code: "22008", message: `date field value out of range: ${field[1]}` };
-    // a cast to numeric(p, s) of a value with more than p - s integer digits
-    if (/Casting value "[^"]*" to type DECIMAL\(\d+,\s*\d+\) failed: value is out of range/.test(message))
-        return { code: "22003", message: "numeric field overflow" };
     // an exact sum past DuckDB's 38 digits (a text read as numeric is summed as a DECIMAL(38,18), src/duck/rewrite.ts summed)
     if (/^Overflow in HUGEINT addition\b/.test(message))
         return { code: "0A000", message: `a sum past the engine's 38 digits is not supported by Mason (${gapMark("numeric-exact-sum")})`, gap: "numeric-exact-sum" };
     // a numeric(p, s) read from a text holding NaN or Infinity (src/duck/rewrite.ts pgInput): DuckDB's decimal has none
-    const special = /Could not convert string ['"](NaN|-?Infinity)['"] to DECIMAL/.exec(message);
-    if (special)
-        return { code: "0A000", message: `a numeric ${special[1]} read from a text is not supported by Mason (${gapMark("numeric-special")})`, gap: "numeric-special" };
+    if (/Could not convert string ['"](NaN|-?Infinity)['"] to DECIMAL/.test(message))
+        return { code: "0A000", message: `a numeric NaN or Infinity read from a text is not supported by Mason (${gapMark("numeric-special")})`, gap: "numeric-special" };
+    // A NUMERIC PAST ITS DIGITS, in every spelling the engine gives (a generated table's run stores one, src/duck/generated.ts
+    // failureWords): a cast to numeric(p, s), of a value or a text, with more than p - s digits before the point, said
+    // with Postgres's DETAIL (and a number constant that does not fit: `12345::numeric(3)`, DuckDB's "Could not cast value
+    // 12345 to DECIMAL(3,0)"); past the 38 digits the engine computes a numeric with, by an operation, a ROUND or a cast
+    const typmod = /(?:Casting value "[^"]*" to type|Could not convert string ['"].*['"] to|^Could not cast value \S+ to) DECIMAL\((\d+),\s*(\d+)\)/.exec(message);
+    if (typmod !== null && Number(typmod[1]) < 38) {
+        return { code: "22003", ...fieldOverflow(Number(typmod[1]), Number(typmod[2])) };
+    }
+    if (typmod !== null || /Overflow in (?:\w+ )?(?:addition|subtraction|multiplication|negation|ROUND|cast)\b.*DECIMAL|Overflow in HUGEINT \w+|out of range for (?:the desired )?DECIMAL|Could not cast value .* to DECIMAL/i.test(message)) {
+        return { code: "22003", message: "numeric field overflow", detail: "The value has more than the 38 digits Mason's engine computes a numeric with." };
+    }
     const conv = /Could not convert string '((?:[^']|'')*)' to (INT16|INT32|INT64|DOUBLE|FLOAT|DATE|TIMESTAMP|UUID|BOOL)/.exec(message);
     if (conv) {
         const t = { INT16: "smallint", INT32: "integer", INT64: "bigint", DOUBLE: "double precision", FLOAT: "real", DATE: "date", TIMESTAMP: "timestamp", UUID: "uuid", BOOL: "boolean" }[conv[2]];
@@ -362,9 +568,10 @@ function postgresWording(kind, message) {
     }
     // a Mason function's own error (src/duck/functions.ts): Postgres's words after the prefix
     // Postgres's own error, from its own function (src/duck/pg-eval.ts): its SQLSTATE, said
-    const pg = /^mason\[([0-9A-Z]{5})\]: (.*)$/s.exec(message);
+    // (and its DETAIL on a line of its own, src/duck/functions.ts `raised`)
+    const pg = /^mason\[([0-9A-Z]{5})\]: (.*?)(?:\nDETAIL: (.*))?$/s.exec(message);
     if (pg)
-        return { code: pg[1], message: pg[2] };
+        return { code: pg[1], message: pg[2], ...(pg[3] === undefined ? {} : { detail: pg[3] }) };
     const own = /^mason: (.*)$/s.exec(message);
     if (own)
         return { code: /^cannot |argument list|key must not be null/.test(own[1]) ? "22023" : /^invalid input syntax/.test(own[1]) ? "22P02" : / is not supported| not supported for type /.test(own[1]) ? "0A000" : "XX000", message: own[1] };
@@ -373,45 +580,27 @@ function postgresWording(kind, message) {
         return { code: "22P02", message };
     return null;
 }
-/** when the wait for a statement to end is said in the log */
-const ENDING_SLOW_MS = 5_000;
-/** One wire session's DuckDB connection, with the session's zone and search_path applied before each statement. Runs
- *  on the executor worker (src/duck/executor-worker.ts), its statements asked by the listener's RemoteRunner. */
-export class SessionRunner {
-    /** the session's slot of the cancel table (src/duck/cancel.ts), which its connection's `__mason_session` names */
-    slot;
-    cancels;
-    /** the number of the statement `run` or `explain` is asked next (the listener's, carried by the request) */
-    next = 0;
-    /** the answers being drained (`streamed`): `close` and the next statement wait for them first */
-    releasing = new Set();
-    /** what runs on the connection now: a statement being started, a chunk being read, an EXPLAIN. `close` waits for it
-     *  too — a connection closed under DuckDB's pending call aborted the process (a stop during a drain) */
-    busy = new Set();
-    /** the answer left open on the connection, if any: what `close` drains before it closes the connection */
-    openAnswer = null;
+export class SessionSetup {
     conn;
-    log;
     zone = null;
     path = null;
     settings = null;
-    constructor(conn, log, slot, cancels) {
-        this.conn = conn;
-        this.log = log;
-        this.slot = slot;
-        this.cancels = cancels;
-    }
-    /** a connection for one Postgres session, in the cancel table's `slot` */
-    static async open(instance, log, slot, cancels) {
-        const conn = await instance.connect();
+    constructor(conn) { this.conn = conn; }
+    /** `name`: what the Mason functions are passed as the session: a slot of the cancel table (`s<n>`, src/duck/cancel.ts)
+     *  for a wire session; any other name is never cancelled through the table (a run is interrupted instead). Set up as a
+     *  session with no role is: UTC, under `public`, no settings, until `settle` says otherwise. */
+    static async open(conn, name) {
         // NULLs sort as Postgres sorts them: last ascending, FIRST descending (DuckDB's default is last both ways). An
         // integer division is the rewrite's `//` where Postgres's operator is integer division (src/duck/rewrite.ts), so
         // DuckDB's `/` stays the float division every other `/` is
         await conn.run("SET default_null_order = 'nulls_last_on_asc_first_on_desc'");
         // every Mason function's first argument (src/duck/functions.ts "every call carries its session")
-        await conn.run(`SET VARIABLE ${SESSION_VARIABLE} = ${quoteLiteral(slotName(slot))}`);
-        return new SessionRunner(conn, log, slot, cancels);
+        await conn.run(`SET VARIABLE ${SESSION_VARIABLE} = ${quoteLiteral(name)}`);
+        const setup = new SessionSetup(conn);
+        await setup.settle(NO_ROLE_SESSION);
+        return setup;
     }
+    /** the session's zone, search_path and settings, each set when it changed: before each statement */
     async settle(session) {
         if (session.timeZone !== this.zone) {
             await this.conn.run(`SET TimeZone = ${quoteLiteral(session.timeZone)}`);
@@ -434,6 +623,43 @@ export class SessionRunner {
             this.settings = settings;
         }
     }
+}
+/** a session with no role, as a generated table is planned and run (src/duck/generated-plan.ts): UTC, under `public`, no
+ *  settings (what reads one is refused there: `current_setting` is "it reads a session") */
+export const NO_ROLE_SESSION = { timeZone: "UTC", searchPath: "public", dateStyle: "ISO, MDY", settings: {} };
+/** when the wait for a statement to end is said in the log */
+const ENDING_SLOW_MS = 5_000;
+/** One wire session's DuckDB connection, with the session's zone and search_path applied before each statement. Runs
+ *  on the executor worker (src/duck/executor-worker.ts), its statements asked by the listener's RemoteRunner. */
+export class SessionRunner {
+    /** the session's slot of the cancel table (src/duck/cancel.ts), which its connection's `__mason_session` names */
+    slot;
+    cancels;
+    /** the number of the statement `run` or `explain` is asked next (the listener's, carried by the request) */
+    next = 0;
+    /** the answers being drained (`streamed`): `close` and the next statement wait for them first */
+    releasing = new Set();
+    /** what runs on the connection now: a statement being started, a chunk being read, an EXPLAIN. `close` waits for it
+     *  too — a connection closed under DuckDB's pending call aborted the process (a stop during a drain) */
+    busy = new Set();
+    /** the answer left open on the connection, if any: what `close` drains before it closes the connection */
+    openAnswer = null;
+    conn;
+    log;
+    setup;
+    constructor(conn, log, slot, cancels, setup) {
+        this.conn = conn;
+        this.log = log;
+        this.slot = slot;
+        this.cancels = cancels;
+        this.setup = setup;
+    }
+    /** a connection for one Postgres session, in the cancel table's `slot` */
+    static async open(instance, log, slot, cancels) {
+        const conn = await instance.connect();
+        return new SessionRunner(conn, log, slot, cancels, await SessionSetup.open(conn, slotName(slot)));
+    }
+    async settle(session) { await this.setup.settle(session); }
     async prepare(sql) {
         try {
             return await this.conn.prepare(sql);
@@ -613,7 +839,7 @@ export class SessionRunner {
             const r = result;
             if (fields !== null && r.columnCount < fields.length)
                 throw new StatementError({ code: "XX000", message: `the engine answered ${r.columnCount} column(s) where Postgres's analysis has ${fields.length}: report it (Mason's rewrite)` }, sql);
-            // Postgres's columns where its analysis gave them (a WITH TIES rank DuckDB adds after them is not sent)
+            // Postgres's columns where its analysis gave them (the exact numerics' cells DuckDB answers after them are not sent)
             const columns = fields !== null
                 ? fields.map((f, i) => pgTypedColumn(f.name, f.oid, r.columnType(i), f.typmod))
                 : Array.from({ length: r.columnCount }, (_, i) => wireColumn(r.columnName(i), r.columnType(i)));
@@ -918,7 +1144,7 @@ function exactRows(computed, rows, base, sql, params) {
             }
             catch (err) {
                 if (err instanceof NumericError)
-                    throw new StatementError({ code: err.sqlstate, message: err.message }, sql, err.gap);
+                    throw new StatementError({ code: err.sqlstate, message: err.message, ...(err.detail === null ? {} : { detail: err.detail }) }, sql, err.gap);
                 throw err;
             }
         }
@@ -933,7 +1159,7 @@ function computedOf(exact, fields) {
     });
 }
 /** whether the column's program reads cell j */
-const programReads = (e, j) => JSON.stringify(e.program).includes(`"key":"${j}"`);
+const programReads = (e, j) => cellsOf(e.program).includes(String(j));
 /** a statement on one line: whole in the log (its ORDER BY, past 160 characters, was what told a slow preview's
  *  cause), cut where it goes to a client in an error's detail */
 const oneLine = (sql, max = Infinity) => { const t = sql.replace(/\s+/g, " ").trim(); return t.length > max ? `${t.slice(0, max - 3)}…` : t; };

package/dist/src/duck/stream.js

@@ -45,6 +45,7 @@
  * unchanged-TOAST update) whose earlier row went in an earlier piece is applied as an UPDATE of the columns it carries,
  * as one whose row the copy holds always is.
  */
+import { errorText, messageOf } from "../lib/secret-mask.js";
 import { decode } from "../cdc/pgoutput.js";
 import { PgConnection, formatLsn, pgIdent, pgLiteral } from "../cdc/pg-wire.js";
 const lit = (s) => `'${s.replace(/'/g, "''")}'`;
@@ -262,6 +263,7 @@ export class ReplicationStream {
     publication;
     target;
     log;
+    errorText;
     flushMs;
     flushRows;
     constructor(o) {
@@ -270,6 +272,7 @@ export class ReplicationStream {
         this.publication = o.publication;
         this.target = o.target;
         this.log = o.log;
+        this.errorText = o.errorText ?? ((e) => errorText(e));
         this.flushMs = o.flushMs ?? 500;
         this.flushRows = o.flushRows ?? PIECE_CHANGES;
         this.acked = this.target.applied();
@@ -289,7 +292,7 @@ export class ReplicationStream {
             this.repl?.sendStatus(this.acked, true);
         }
         catch (e) {
-            this.log(`replication: could not ask the source for its position (${e.message})`);
+            this.log(`replication: could not ask the source for its position (${this.errorText(e)})`);
         }
     }
     async run() {
@@ -302,7 +305,7 @@ export class ReplicationStream {
                 if (this.stopping)
                     break;
                 const wait = Math.min(30_000, 1_000 * 2 ** Math.min(attempts++, 5));
-                this.log(`replication: the source is away (${e.message}); trying again in ${wait / 1000} s`);
+                this.log(`replication: the source is away (${this.errorText(e)}); trying again in ${wait / 1000} s`);
                 await new Promise((r) => setTimeout(r, wait));
                 continue;
             }
@@ -329,7 +332,7 @@ export class ReplicationStream {
                     this.repl = null;
                     return e;
                 }
-                this.log(`replication: the source went away (${e.message}); reconnecting`);
+                this.log(`replication: the source went away (${this.errorText(e)}); reconnecting`);
             }
             finally {
                 this.repl?.close();
@@ -387,7 +390,7 @@ export class ReplicationStream {
                 repl.sendStatus(this.acked, true);
             }
             catch (e) {
-                this.log(`replication: the status update could not be sent (${e.message})`);
+                this.log(`replication: the status update could not be sent (${this.errorText(e)})`);
             }
         }, 10_000);
         const command = `START_REPLICATION SLOT ${pgIdent(this.slot)} LOGICAL 0/0 (proto_version '1', publication_names ${pgLiteral(`"${this.publication.replace(/"/g, '""')}"`)})`;
@@ -518,6 +521,7 @@ export function pgTime(us) {
 }
 /** the source being away — closed, refused, reset, a shutdown — which a reconnect outlasts; anything else is the stream's */
 function away(e) {
-    const m = e instanceof Error ? e.message : String(e);
+    // a decision: the raw words (src/lib/secret-mask.ts messageOf), never the ones Mason repeats
+    const m = messageOf(e);
     return /connection closed|ECONNREFUSED|ECONNRESET|EPIPE|ETIMEDOUT|timed out|terminating connection|the database system is (shutting down|starting up)|server closed|is active for PID/i.test(m);
 }