@mason-data/cli 0.6.2 → 0.8.0

package/dist/src/duck/store.js

@@ -16,23 +16,23 @@
  * (src/duck/sync.ts attach), so the store itself opens offline.
  */
 import * as os from "node:os";
-import { setTimeout as sleep } from "node:timers/promises";
 import { DuckDBInstance, version } from "@duckdb/node-api";
 import { Executor } from "./executor.js";
 import { beginStatement, cancelStatement, slotName, statementCancelled } from "./cancel.js";
 import { StatementError } from "../api/postgres/server.js";
 import { duckParameters, missingRelationHint, planDuck, quoteDuckKeywords } from "./plan.js";
 import { VERIFIED } from "./verified.js";
-import { msSinceCall, registerFunctions, SESSION_VARIABLE } from "./functions.js";
+import { registerFunctions, SESSION_VARIABLE, SETTINGS_VARIABLE } from "./functions.js";
 import { duckCollation } from "./rewrite.js";
 import { decodeLibcOrder, encodeLibcOrder } from "./libc-order.js";
 import { dataRow } from "../api/postgres/protocol.js";
 import { cellText, exactColumn, pgTypedColumn, wireColumn } from "./types.js";
-import { gapMark } from "./gaps.js";
+import { gapIn, gapMark } from "./gaps.js";
 import { cellsOf, evaluateText, NumericError } from "../lib/pg-numeric.js";
-import { explainJson, explainText, planNodes, scannedTables } from "./explain.js";
+import { explainJson, explainText, planNodes, profiled, qualifiedName, scannedTables } from "./explain.js";
 import * as fs from "node:fs";
 import * as path from "node:path";
+import { extensionSettings, findExtensions } from "./extensions.js";
 export const MASON_SCHEMA = "_mason";
 /** the DuckDB the prototype runs (`@duckdb/node-api`'s bundled library) */
 export const DUCKDB_VERSION = version();
@@ -49,8 +49,19 @@ export const DUCKDB_VERSION = version();
  * disk at 125 GB, against 0.26 GB forced; a sample of its text columns 4.5× smaller, the integer, UUID and timestamp
  * columns keeping their own methods, BitPacking and RLE, since forcing only touches what ZSTD can store). The cost of
  * reading a ZSTD column is not measured. tests/duck-wire.test.ts holds a string column to ZSTD.
+ *
+ * `lambda_syntax: ENABLE_SINGLE_ARROW`: the rewrite is a Postgres parse tree, printed by Postgres's deparser, which
+ * can spell DuckDB's lambda only as `x -> f(x)` (Postgres's `->` operator), never as `lambda x: f(x)`; DuckDB 2.0
+ * refuses that spelling by default. Two rewrites need a lambda: a jsonb's order under ICU or a libc locale, which maps
+ * DuckDB's `icu_sort_key` over the document's strings (src/duck/rewrite.ts `jsonbKey`; tests/duck-jsonb-order.test.ts
+ * holds the key under every collation), and a LEFT lateral's unnest of an expression, which computes the expression
+ * once with `list_transform(list_value(e), l -> CASE … l … END)[1]` (`rangeFunction`; tests/duck-wire.test.ts). A `->`
+ * that is not a lambda function's argument is still the json operator.
+ *
+ * Beside them, the copy's extensions (src/duck/extensions.ts `extensionSettings`): with Mason's package for this
+ * platform, `extension_directory` is that package and nothing is ever downloaded; without one, DuckDB's defaults.
  */
-const INSTANCE_SETTINGS = { late_materialization_max_rows: "10000", force_compression: "zstd" };
+const INSTANCE_SETTINGS = { late_materialization_max_rows: "10000", force_compression: "zstd", lambda_syntax: "ENABLE_SINGLE_ARROW" };
 /**
  * DUCKDB'S MEMORY, BOUNDED FOR THIS PROCESS: DuckDB's default is 80% of the machine's RAM, in a process
  * that also holds V8's heap and two PGlites, which ends with the system killing every session and the sync together.
@@ -109,8 +120,8 @@ export async function openDuck(file, settings) {
     catch (e) {
         const m = e.message;
         throw new Error(/lock/i.test(m)
-            ? `DuckDB could not open ${file}: another process holds it (${m}). Is another mason start running for this project? mason status says`
-            : `DuckDB could not open ${file}: ${m}. mason destroy deletes the copy, and the next mason start copies again`);
+            ? `Mason could not open the synced data ${file}: another process holds it (${m}). Is another mason start running for this project? mason status says`
+            : `Mason could not open the synced data ${file}: ${m}. mason destroy deletes the synced data, and the next mason start syncs again`);
     }
     const writer = await instance.connect();
     await writer.run(`CREATE SCHEMA IF NOT EXISTS ${MASON_SCHEMA}`);
@@ -131,27 +142,32 @@ export class DuckStore {
     executor;
     closed = null;
     closeGraceMs;
-    constructor(file, executor, memoryLimit, closeGraceMs) {
+    /** where the copy's extensions come from: Mason's package for this platform, or DuckDB's install (src/duck/extensions.ts) */
+    extensions;
+    constructor(file, executor, memoryLimit, extensions, closeGraceMs) {
         this.file = file;
         this.executor = executor;
         this.memoryLimitBytes = memoryLimit;
+        this.extensions = extensions;
         this.closeGraceMs = closeGraceMs;
         this.writer = executor.writer();
         this.instance = { connect: () => executor.connect() };
     }
     /** `silentMs` and `closeGraceMs`: the executor's bounds (src/duck/executor.ts SILENT_MS, CLOSE_GRACE_MS), shorter only
-     *  in a test */
+     *  in a test; `workerFile`, the executor worker's module, another only in a test; `extensions`, this machine's
+     *  (`findExtensions`) unless a test names a package */
     static async open(file, log, options = {}) {
+        const extensions = options.extensions ?? findExtensions();
         const memoryLimit = options.memoryLimit ?? DEFAULT_MEMORY_LIMIT;
         if (!MEMORY_LIMIT_RE.test(memoryLimit))
             throw new Error(`MASON_DUCKDB_MEMORY=${JSON.stringify(memoryLimit)} is not an amount of memory: write it as 8GB or 512MiB (unset, half the machine's RAM: ${DEFAULT_MEMORY_LIMIT})`);
         const where = path.resolve(file);
         const before = opened.get(where);
         if (before !== undefined && before.closed === null)
-            throw new Error(`DuckDB could not open ${file}: this process has it open already (Mason bug: report it)`);
+            throw new Error(`Mason could not open the synced data ${file}: this process has it open already (Mason bug: report it)`);
         // a copy whose thread was given up on may still hold the file: a second instance of it would lose writes
         if (before?.abandoned != null)
-            throw new Error(`DuckDB could not open ${file}: this process gave up on the copy's thread before (${before.abandoned}), which may still hold it; start mason again (a new process opens it)`);
+            throw new Error(`Mason could not open the synced data ${file}: this process gave up on the engine's thread before (${before.abandoned}), which may still hold it; start mason again (a new process opens it)`);
         const mine = { closed: null, abandoned: null };
         opened.set(where, mine);
         let executor;
@@ -159,15 +175,15 @@ export class DuckStore {
             if (before !== undefined)
                 await before.closed;
             // every connection's, the sync's and each session's
-            executor = await Executor.start({ file, settings: { ...INSTANCE_SETTINGS, memory_limit: memoryLimit, temp_directory: `${file}.tmp` }, log,
-                ...(options.silentMs !== undefined ? { silentMs: options.silentMs } : {}) });
+            executor = await Executor.start({ file, settings: { ...INSTANCE_SETTINGS, ...extensionSettings(extensions), memory_limit: memoryLimit, temp_directory: `${file}.tmp` }, log,
+                ...(options.silentMs !== undefined ? { silentMs: options.silentMs } : {}), ...(options.workerFile !== undefined ? { workerFile: options.workerFile } : {}) });
         }
         catch (e) {
             if (opened.get(where) === mine)
                 opened.delete(where);
             throw e;
         }
-        const store = new DuckStore(file, executor, memoryLimitBytes(memoryLimit), options.closeGraceMs);
+        const store = new DuckStore(file, executor, memoryLimitBytes(memoryLimit), extensions, options.closeGraceMs);
         // closed: the file may be opened again — unless its thread was given up on (`abandoned`), which keeps it here
         store.closing = (p) => { mine.closed = p; void p.then(() => { if (executor.abandoned !== null)
             mine.abandoned = executor.abandoned;
@@ -176,10 +192,12 @@ export class DuckStore {
         return store;
     }
     closing = () => undefined;
-    /** called once if the copy's thread dies while the run uses it (src/duck/executor.ts): the run exits with the reason */
+    /** called once if the engine's thread dies while the run uses it (src/duck/executor.ts): the run exits with the reason */
     onCrash(f) { this.executor.onCrash(f); }
-    /** how many vectors each Mason function was called with so far, on the copy's thread (a test's, a measurement's) */
+    /** how many vectors each Mason function was called with so far, on the engine's thread (a test's, a measurement's) */
     functionCalls() { return this.executor.functionCalls(); }
+    /** how many rows each Mason function was handed so far, on the engine's thread (a plan's cost in calls) */
+    functionRows() { return this.executor.functionRows(); }
     /** a connection for one Postgres session (on the executor worker: SessionRunner) */
     runner() { return this.executor.runner(); }
     /**
@@ -211,11 +229,15 @@ export class DuckStore {
                 throw new Error(`the database orders text by the libc collation ${collation.collate}, and its order was not read (Mason bug: report it)`);
             await this.executor.registerLibcOrder(libc);
         }
+        // a jsonb's stored key is this collation's (src/duck/kept.ts `orderBy`)
+        await kept?.orderBy(c, c.kind === "libc" ? libc : undefined);
         return {
             collation: c,
             open: () => this.runner(),
-            plan: (catalog, text, stmt, base, paramOids, searchPath, timeZone, userTypes) => planDuck(catalog, text, stmt, base, paramOids, searchPath, timeZone, c, verified, userTypes, copies, 0, kept ?? null),
+            plan: (catalog, text, stmt, base, paramOids, searchPath, timeZone, userTypes, wireBound, grants) => planDuck(catalog, text, stmt, base, paramOids, searchPath, timeZone, c, verified, userTypes, copies, 0, kept ?? null, wireBound ?? false, grants),
             parameters: duckParameters,
+            // the sync's state, which EXPLAIN says how each copy is kept current from (src/duck/sync.ts KeptCurrent)
+            copies,
             // a stored JSON path ready or gone (src/duck/kept.ts): the plans kept are made again
             ...(kept !== undefined ? { generation: () => kept.generation() } : {}),
         };
@@ -260,21 +282,23 @@ export function pgErrorOf(err, sql) {
     const kind = /^([A-Za-z ]+) Error: /.exec(full)?.[1] ?? "";
     const message = full.replace(/^[A-Za-z ]+ Error: /, "").replace(/\n\n?LINE \d+:[\s\S]*$/, "").trim();
     const pg = postgresWording(kind, message);
+    // a refusal raised inside DuckDB (a Mason function's, the rewrite's error() call) crossed it as its text alone: the
+    // entry its text names (gapIn), read here, the one place that text becomes a statement's error
     if (pg !== null)
-        return new StatementError({ ...pg, detail: `DuckDB: ${message}` }, sql);
+        return new StatementError({ code: pg.code, message: pg.message, detail: `the engine: ${message}` }, sql, pg.gap ?? (pg.code === "0A000" ? gapIn(pg.message) : null));
     const fn = /(?:Scalar|Aggregate|Table) Function with name "?([A-Za-z_][\w]*)"? does not exist/.exec(message)?.[1];
     if (fn !== undefined) {
         // what reaches DuckDB was verified against Postgres (src/duck/verified.ts), so a function DuckDB lacks is a gap in
         // Mason's rewrite, never something for the client to write in DuckDB's dialect
-        return new StatementError({ code: "0A000", message: `function ${fn}() is not supported: Mason answers SQL with DuckDB, which has no ${fn}`,
-            hint: "this is a gap in Mason's translation: report it with the statement (docs/SQL-SUPPORT.md lists what is known)", detail: `DuckDB: ${message}` });
+        return new StatementError({ code: "0A000", message: `function ${fn}() is not supported: Mason's engine has no ${fn}`,
+            hint: "this is a gap in Mason's translation: report it with the statement (docs/SQL-SUPPORT.md lists what is known)", detail: `the engine: ${message}` });
     }
     const table = /Table with name "?([^"!\s]+)"? does not exist/.exec(message)?.[1];
     if (table !== undefined) {
         // DuckDB's miss comes after the catalog held the table (the plan passed): one dropped as no longer added, whose way
         // back is `mason add`, so no copy state is read
         return new StatementError({ code: "42P01", message: `relation "${table}" does not exist`,
-            hint: missingRelationHint(table, []), detail: `DuckDB: ${message}` });
+            hint: missingRelationHint(table, []), detail: `the engine: ${message}` });
     }
     const code = kind === "Parser" ? "42601"
         : kind === "Catalog" ? "42P01"
@@ -294,13 +318,13 @@ export function pgErrorOf(err, sql) {
                                                     : kind === "Invalid Input" ? "22023"
                                                         : "XX000";
     return new StatementError({ code, message: code === "57014" ? "canceling statement due to user request"
-            : code === "53200" ? "out of memory: this statement needs more than the memory Mason's DuckDB may use, in a part that cannot spill to disk" : message,
+            : code === "53200" ? "out of memory: this statement needs more than the memory Mason's engine may use, in a part that cannot spill to disk" : message,
         // DuckDB's own advice is SET, which a client of Mason cannot do: Mason's way out instead
         ...(code === "53200" ? { hint: "ask for less at once (a filter, a LIMIT, fewer or narrower columns, a coarser GROUP BY), or raise MASON_DUCKDB_MEMORY (8GB, 24GB) and start mason again; nothing else stopped" } : {}),
         // what Mason ran (the statement as rewritten for DuckDB, src/duck/rewrite.ts): a report of the error needs it
-        detail: `DuckDB${kind ? ` ${kind} Error` : ""} (Mason answers SQL with DuckDB): ${message}${sql === "" ? "" : `; DuckDB ran: ${oneLine(sql, 2000)}`}`,
+        detail: `the engine${kind ? ` ${kind} error` : ""}: ${message}${sql === "" ? "" : `; Mason ran: ${oneLine(sql, 2000)}`}`,
         // Postgres accepted the GROUP BY (it analysed the statement first): DuckDB refusing it is Mason's rewrite's gap
-        ...(code === "42803" ? { hint: "Postgres accepts this GROUP BY and Mason's rewrite did not carry it to DuckDB: report it with the statement; adding the column to the GROUP BY answers meanwhile" } : {}),
+        ...(code === "42803" ? { hint: "Postgres accepts this GROUP BY and Mason's rewrite did not carry it to the engine: report it with the statement; adding the column to the GROUP BY answers meanwhile" } : {}),
     }, sql);
 }
 /** The errors DuckDB raises for what Postgres raises too, in Postgres's code and words (a tool or a person matches them). */
@@ -326,11 +350,11 @@ function postgresWording(kind, message) {
         return { code: "22003", message: "numeric field overflow" };
     // an exact sum past DuckDB's 38 digits (a text read as numeric is summed as a DECIMAL(38,18), src/duck/rewrite.ts summed)
     if (/^Overflow in HUGEINT addition\b/.test(message))
-        return { code: "0A000", message: `a sum past DuckDB's 38 digits is not supported by Mason (${gapMark("numeric-exact-sum")})` };
+        return { code: "0A000", message: `a sum past the engine's 38 digits is not supported by Mason (${gapMark("numeric-exact-sum")})`, gap: "numeric-exact-sum" };
     // a numeric(p, s) read from a text holding NaN or Infinity (src/duck/rewrite.ts pgInput): DuckDB's decimal has none
     const special = /Could not convert string ['"](NaN|-?Infinity)['"] to DECIMAL/.exec(message);
     if (special)
-        return { code: "0A000", message: `a numeric ${special[1]} read from a text is not supported by Mason (${gapMark("numeric-special")})` };
+        return { code: "0A000", message: `a numeric ${special[1]} read from a text is not supported by Mason (${gapMark("numeric-special")})`, gap: "numeric-special" };
     const conv = /Could not convert string '((?:[^']|'')*)' to (INT16|INT32|INT64|DOUBLE|FLOAT|DATE|TIMESTAMP|UUID|BOOL)/.exec(message);
     if (conv) {
         const t = { INT16: "smallint", INT32: "integer", INT64: "bigint", DOUBLE: "double precision", FLOAT: "real", DATE: "date", TIMESTAMP: "timestamp", UUID: "uuid", BOOL: "boolean" }[conv[2]];
@@ -349,12 +373,8 @@ function postgresWording(kind, message) {
         return { code: "22P02", message };
     return null;
 }
-/** how long no call of a session must have been answered before a result of its that did not end is closed: a task
- *  waiting for a call is answered within a turn of the worker's loop (microseconds when it is free), and the work a task
- *  does between two calls is one vector's (2,048 rows) — milliseconds; 50 ms leaves an order of magnitude over both */
-const CALLS_QUIET_MS = 50;
-/** when the wait for a statement's calls to stop is said in the log */
-const CALLS_SLOW_MS = 5_000;
+/** when the wait for a statement to end is said in the log */
+const ENDING_SLOW_MS = 5_000;
 /** One wire session's DuckDB connection, with the session's zone and search_path applied before each statement. Runs
  *  on the executor worker (src/duck/executor-worker.ts), its statements asked by the listener's RemoteRunner. */
 export class SessionRunner {
@@ -374,6 +394,7 @@ export class SessionRunner {
     log;
     zone = null;
     path = null;
+    settings = null;
     constructor(conn, log, slot, cancels) {
         this.conn = conn;
         this.log = log;
@@ -405,6 +426,13 @@ export class SessionRunner {
             await this.conn.run(`SET search_path = ${quoteLiteral(path)}`);
             this.path = path;
         }
+        // what `current_setting(name)` reads (src/duck/rewrite.ts `settingRead`): the session's settings, as one map
+        const settings = JSON.stringify(session.settings);
+        if (settings !== this.settings) {
+            const entries = Object.entries(session.settings).map(([k, v]) => `${quoteLiteral(k)}: ${quoteLiteral(v)}`).join(", ");
+            await this.conn.run(`SET VARIABLE ${SETTINGS_VARIABLE} = MAP {${entries}}::MAP(VARCHAR, VARCHAR)`);
+            this.settings = settings;
+        }
     }
     async prepare(sql) {
         try {
@@ -431,8 +459,9 @@ export class SessionRunner {
         return p;
     }
     /** `run`, and where its time goes as it is read: DuckDB's (its prepare, which plans; each chunk it computes) and
-     *  Mason's (the exact numerics, the encoding), ms — what an EXPLAIN ANALYZE says */
-    async streamed(planned, params, session, signal, fields, exact) {
+     *  Mason's (the exact numerics, the encoding), ms — what an EXPLAIN ANALYZE says. `profile`: the file DuckDB's
+     *  profiler writes this statement's profile to (EXPLAIN ANALYZE), turned on here, after the settle */
+    async streamed(planned, params, session, signal, fields, exact, profile = null) {
         const sql = planned.text;
         const seq = this.next;
         await Promise.all([...this.releasing]);
@@ -445,18 +474,28 @@ export class SessionRunner {
         catch (e) {
             throw pgErrorOf(e, sql);
         }
+        // THE PROFILER IS ON FOR THE STATEMENT ALONE: DuckDB writes the profile of the LAST statement it profiled on the
+        // connection, so it is turned on after every other statement Mason runs here (the settle's probe of the schemas,
+        // turned on before it, was reported as the plan of a count(*) that wrote no profile of its own), and the caller turns
+        // it off before it runs another (`explained`, which also holds the profile to the statement's text)
+        if (profile !== null) {
+            try {
+                await this.conn.run("SET enable_profiling = 'json'");
+                await this.conn.run(`SET profiling_output = ${quoteLiteral(profile)}`);
+            }
+            catch (e) {
+                throw pgErrorOf(e, sql);
+            }
+        }
         const timing = { prepareMs: 0, duckdbMs: 0, masonMs: 0 };
         const t0 = performance.now();
         const p = await this.prepare(sql);
         timing.prepareMs = performance.now() - t0;
         // HOW A STATEMENT IS CANCELLED (src/duck/functions.ts "every call carries its session"; the test
-        // server's hung process). One that calls no Mason function is interrupted, and its result closed at once: the
-        // close is what cancels the executor's tasks, and a close put off lets the session's next statement take the
-        // connection first, after which the close waits for the cancelled query to run to its end (38 s, in
-        // tests/duck-wire.test.ts "statement_timeout"). One that does is cancelled through its session — its next call
-        // raises — AND interrupted (its tasks stop at their next chunk, so a call-free rest past its last call stops too),
-        // and its result is closed only once DuckDB has handed it back (the fetch's own path, or `drain`) and its calls
-        // have stopped (`cleanUp`: a task still inside a call waits for this thread, which a close would hold). Whether it calls
+        // server's hung process). It is interrupted — its tasks stop at their next chunk — and, if it calls a Mason
+        // function, cancelled through its session as well: its next call raises, so a task of it does not call on. Its
+        // result is closed only once DuckDB ended the query, on a thread that is not this one (`cleanUp`: a task still
+        // inside a call waits for this thread, which a close would hold). Whether it calls
         // one is the PLAN's (src/duck/plan.ts PlannedSql `functions`, read off the rewritten tree), never a match on its
         // text: a stored JSON path's column and the view of a copy's visible columns are named `__mason_…` too, and
         // read as calls, every `SELECT *` over such a copy was cancelled only by a call it never made (a review
@@ -481,7 +520,20 @@ export class SessionRunner {
         const started = Date.now();
         let total = 0;
         let result = null;
-        let closed = false, fetching = false, released = false;
+        let closed = false, fetching = false, released = false, said = false;
+        /** DuckDB ended the query: a fetch failed, or read its end (`cleanUp` says why that is what `release` waits for) */
+        let ended = false;
+        let ending = false;
+        // ITS LINE IS SAID WHEN IT ENDS, before its client hears the end, never when its result is released: that waits for
+        // DuckDB to end it (`cleanUp`), by when its client is gone and the next session's statements are running, and a
+        // reader of the log took the late line for theirs (`tests/duck-wire.test.ts`, 1 run in 100 under load). The
+        // worker's port keeps the order: a line posted before the answer is read before it
+        const say = () => {
+            if (said)
+                return;
+            said = true;
+            this.log(`statement: ${total} row(s) in ${Date.now() - started} ms — ${oneLine(sql)}`);
+        };
         const release = () => {
             if (released)
                 return;
@@ -491,52 +543,32 @@ export class SessionRunner {
             signal.removeEventListener("abort", cancel);
             result?.closeSync();
             p.destroySync();
-            this.log(`statement: ${total} row(s) in ${Date.now() - started} ms — ${oneLine(sql)}`);
+            say();
         };
-        // A CALLING STATEMENT THAT DID NOT RUN TO ITS END (it failed, its cancel raised, it was closed early) IS CLOSED
-        // ONLY ONCE ITS CALLS HAVE STOPPED (the suite hung for good, 0 % CPU). DuckDB hands back a failed chunk
-        // while another of the query's tasks may still wait INSIDE a call — for this thread, which runs every Mason
-        // function — and closing the result waits for that task (Executor::CancelTasks): this thread waited for the task,
-        // the task for this thread, and every later request to the copy with them (`cleanUp`). One that calls no function,
-        // or one read to its end, has no task left to wait for and is released at once
-        const releaseAfterCleanup = () => {
-            if (!callsFunctions || released) {
-                release();
+        // A STATEMENT THAT DID NOT RUN TO ITS END (it failed, its cancel raised, it was closed early) IS CLOSED ONLY ONCE
+        // DuckDB ENDED IT OFF THIS THREAD (`cleanUp`), the session's next statement waiting for that (`releasing`). One read
+        // to its end has no task left, and is released at once
+        const endOffThisThread = () => {
+            if (released || ending)
                 return;
-            }
-            const done = (async () => { await this.cleanUp(seq, sql); release(); })();
+            ending = true;
+            const done = (async () => { await this.cleanUp(seq, sql, callsFunctions, ended ? null : result); release(); })();
             this.releasing.add(done);
             void done.finally(() => this.releasing.delete(done));
         };
-        // a result closed before its end while no fetch is in flight: a statement calling no function is released at
-        // once (the close cancels the executor); one that does is read on until DuckDB hands it back refused (its next
-        // call raises) or read to its end, then released — the session's next statement waits for that (`releasing`)
+        // it ended with an answer its client is about to hear: said now
+        const releaseAfterCleanup = () => { if (!released)
+            say(); endOffThisThread(); };
+        // a result closed before its end while no fetch is in flight: no end its client hears, so its line is its release's
         const drain = () => {
-            const r = result;
-            if (!callsFunctions || r === null) {
-                release();
-                return;
-            }
-            const done = (async () => {
-                // read on until the next call raises or the result ends — an EMPTY chunk, as `fetch` reads it: past its end a
-                // streamed result hands back empty chunks for ever, never null, and a result closed after its last row but
-                // before its end was read here without end, a thread spinning and the session's close waiting on it
-                try {
-                    for (let c = await r.fetchChunk(); c !== null && c.rowCount > 0; c = await r.fetchChunk()) { /* its rows are nobody's */ }
-                }
-                catch (e) {
-                    // the cancel's own refusal (its next call raised, 57014) is what the drain waits for. Anything else is the
-                    // statement's real error, met after the listener let go of its answer: no one reads it any more, and a
-                    // rethrow here would reach no caller (it would end this thread, and with it every session), so it is said
-                    if (pgErrorOf(e, sql).fields.code !== "57014")
-                        this.log(`a statement closed before its end failed while it was drained: ${e instanceof Error ? e.message : String(e)} — ${oneLine(sql)}`);
-                    // it failed, so a task of it may still be inside a call: closed once its calls have stopped
-                    await this.cleanUp(seq, sql);
-                }
-                release();
-            })();
-            this.releasing.add(done);
-            void done.finally(() => this.releasing.delete(done));
+            // CLOSED WHILE DuckDB STILL STARTS IT (its `stream()` pending: an aggregate computes there whole): nothing is
+            // released under the pending start — that destroyed the prepared statement under it and, its release done, let the
+            // start's own failure skip `cleanUp`, so the connection and the instance closed while its other tasks still had
+            // calls queued for this thread, and the thread's end met them (Linux CI: `FATAL ERROR: Error::Error
+            // napi_define_properties`, 7 runs in 30 of a stop under a calling count). The start's own path releases it: its
+            // failure through `releaseAfterCleanup`, its answer drained once handed back (below `stream()`)
+            if (result !== null)
+                endOffThisThread();
         };
         // an early end by the listener (a portal closed half read, the session gone): cancelled, and released — by the
         // fetch in flight once DuckDB hands the chunk back, else by `drain`
@@ -565,10 +597,22 @@ export class SessionRunner {
             catch (e) {
                 throw failure(e);
             }
+            // CLOSED WHILE IT STARTED (`drain` above), now that DuckDB handed it back, and drained. One that calls no function
+            // is told the cancel here, its release done first, before a column of a destroyed result is asked (DuckDB
+            // answers 0 columns for one: a review finding, an XX000 "report it" where the session had closed); one that calls
+            // holds its result open until DuckDB ended it, so its answer below is built from a live result and its fetch
+            // tells the cancel
+            if (closed) {
+                drain();
+                if (!callsFunctions) {
+                    await Promise.all([...this.releasing]);
+                    throw cancelled();
+                }
+            }
             timing.duckdbMs += performance.now() - t1;
             const r = result;
             if (fields !== null && r.columnCount < fields.length)
-                throw new StatementError({ code: "XX000", message: `DuckDB answered ${r.columnCount} column(s) where Postgres's analysis has ${fields.length}: report it (Mason's rewrite)` }, sql);
+                throw new StatementError({ code: "XX000", message: `the engine answered ${r.columnCount} column(s) where Postgres's analysis has ${fields.length}: report it (Mason's rewrite)` }, sql);
             // Postgres's columns where its analysis gave them (a WITH TIES rank DuckDB adds after them is not sent)
             const columns = fields !== null
                 ? fields.map((f, i) => pgTypedColumn(f.name, f.oid, r.columnType(i), f.typmod))
@@ -598,11 +642,13 @@ export class SessionRunner {
                             chunk = await this.track(r.fetchChunk());
                         }
                         catch (e) {
+                            ended = true;
                             throw failure(e);
                         }
                         finally {
                             fetching = false;
                         }
+                        ended = chunk === null || chunk.rowCount === 0;
                         if (closed) {
                             drain();
                             return null;
@@ -649,57 +695,35 @@ export class SessionRunner {
      * EXPLAIN of a plan (src/duck/explain.ts). Without ANALYZE: DuckDB's `EXPLAIN (FORMAT JSON)` of the planned SQL, its
      * parameters bound. With it: the statement RUN as `run` runs it — every chunk read, its exact numerics computed and
      * encoded, none sent — with DuckDB's profiler writing each operator's rows and time to a file of its own, read after.
-     * Then the copies its scans read: their rows, and their last poll (`_mason.poll_state`).
+     * Then the copies its scans read: their rows, and how each is kept current as the sync says it (`how.copies`).
      */
     explain(sql, params, session, signal, how) {
         return this.track(this.explained(sql, params, session, signal, how));
     }
     async explained(sql, params, session, signal, how) {
-        await Promise.all([...this.releasing]);
         const o = how.options;
+        // A STATEMENT THAT READS A FILTERED TABLE IS NEVER SHOWN TO THE ENGINE: its plan says what it proved from the table's
+        // statistics (docs/DESIGN.md "Roles: rows per user"), so the tables are the statement's and its filters' (a through:'s
+        // parents), and the rest is Mason's plan
+        if ((how.rowFilters ?? []).length > 0) {
+            const tables = [...new Set([...(how.reads ?? []), ...(how.rowFilters ?? []).map((f) => f.table)])].map((t) => { const dot = t.indexOf("."); return { schema: t.slice(0, dot), name: t.slice(dot + 1) }; });
+            const explained = { copies: await this.copies(tables, how.copies, how.rowFilters ?? []), sql: oneLine(sql.text), computed: computedOf(how.exact, how.fields),
+                functions: [...sql.functions], plan: [], filtered: true };
+            return o.format === "json" ? [explainJson(explained, o)] : explainText(explained, o);
+        }
+        await Promise.all([...this.releasing]);
         let plan;
         let analyzed;
-        if (!o.analyze) {
-            try {
-                await this.settle(session);
-            }
-            catch (e) {
-                throw pgErrorOf(e, sql.text);
-            }
-            const p = await this.prepare(`EXPLAIN (FORMAT JSON) ${sql.text}`);
-            const interrupt = () => this.conn.interrupt();
-            signal.addEventListener("abort", interrupt);
-            try {
-                if (signal.aborted)
-                    throw failureOf(signal, null, sql.text);
-                params.forEach((v, i) => { if (v === null)
-                    p.bindNull(i + 1);
-                else
-                    p.bindVarchar(i + 1, v); });
-                let r;
-                try {
-                    r = await p.runAndReadAll();
-                }
-                catch (e) {
-                    throw failureOf(signal, e, sql.text);
-                }
-                plan = planNodes(JSON.parse(String(r.getRowsJS()[0]?.[1] ?? "[]")));
-            }
-            finally {
-                signal.removeEventListener("abort", interrupt);
-                p.destroySync();
-            }
-        }
+        if (!o.analyze)
+            plan = await this.plainPlan(sql, params, session, signal);
         else {
             const dir = fs.mkdtempSync(path.join(os.tmpdir(), "mason-explain-"));
             const file = path.join(dir, "profile.json");
             try {
                 let rows = 0;
                 let timing;
-                await this.conn.run("SET enable_profiling = 'json'");
                 try {
-                    await this.conn.run(`SET profiling_output = ${quoteLiteral(file)}`);
-                    const s = await this.streamed(sql, params, session, signal, how.fields, how.exact);
+                    const s = await this.streamed(sql, params, session, signal, how.fields, how.exact, file);
                     timing = s.timing;
                     try {
                         for (let chunk = await s.answer.fetch([], { timeZone: session.timeZone }); chunk !== null; chunk = await s.answer.fetch([], { timeZone: session.timeZone }))
@@ -710,70 +734,158 @@ export class SessionRunner {
                     }
                 }
                 finally {
+                    await Promise.all([...this.releasing]);
                     await this.conn.run("PRAGMA disable_profiling");
                 }
-                // the profile is the statement's: written when its result was read to its end, after every other statement here
-                plan = planNodes(JSON.parse(fs.readFileSync(file, "utf8")).operator ?? []);
-                analyzed = { rows, planning: { ...how.planning, duckdbMs: timing.prepareMs }, execution: { duckdbMs: timing.duckdbMs, masonMs: timing.masonMs } };
+                // THE PROFILE IS HELD TO THE STATEMENT, never assumed its: DuckDB writes one when the result is read to its end,
+                // naming the statement it timed. None: DuckDB ran no operator (it answered while planning — a count(*) of a
+                // whole table is the table's row count), and its plan is its EXPLAIN, the rows the ones it answered
+                const profile = fs.existsSync(file) ? profiled(JSON.parse(fs.readFileSync(file, "utf8"))) : null;
+                if (profile !== null && profile.sql !== sql.text) {
+                    throw new StatementError({ code: "XX000", message: `the engine's profile of an EXPLAIN ANALYZE is of another statement (${oneLine(profile.sql ?? "none named")}): report it (Mason's EXPLAIN)` }, sql.text);
+                }
+                if (profile !== null)
+                    plan = profile.plan;
+                else {
+                    plan = await this.plainPlan(sql, params, session, signal);
+                    if (plan[0] !== undefined)
+                        plan[0].actualRows = rows;
+                }
+                analyzed = { rows, profiled: profile !== null, planning: { ...how.planning, duckdbMs: timing.prepareMs }, execution: { duckdbMs: timing.duckdbMs, masonMs: timing.masonMs } };
             }
             finally {
                 fs.rmSync(dir, { recursive: true, force: true });
             }
         }
         // what it calls is the plan's (src/duck/plan.ts PlannedSql), the list that decides how it is cancelled
-        const explained = { copies: await this.copies(scannedTables(plan)), sql: oneLine(sql.text), computed: computedOf(how.exact, how.fields),
+        const explained = { copies: await this.copies(scannedTables(plan), how.copies, how.rowFilters ?? []), sql: oneLine(sql.text), computed: computedOf(how.exact, how.fields),
             functions: [...sql.functions], plan, ...(analyzed === undefined ? {} : { analyzed }) };
         return o.format === "json" ? [explainJson(explained, o)] : explainText(explained, o);
     }
-    /** each copy's rows and last poll (none recorded: a copy made whole, not polled yet) */
-    async copies(tables) {
-        const polls = (await this.conn.runAndReadAll(`SELECT count(*) FROM duckdb_tables() WHERE database_name = current_database() AND schema_name = '${MASON_SCHEMA}' AND table_name = 'poll_state'`)).getRowsJS()[0][0] !== 0n;
+    /** DuckDB's `EXPLAIN (FORMAT JSON)` of the planned SQL, its parameters bound: its plan, estimates only */
+    async plainPlan(sql, params, session, signal) {
+        try {
+            await this.settle(session);
+        }
+        catch (e) {
+            throw pgErrorOf(e, sql.text);
+        }
+        const p = await this.prepare(`EXPLAIN (FORMAT JSON) ${sql.text}`);
+        const interrupt = () => this.conn.interrupt();
+        signal.addEventListener("abort", interrupt);
+        try {
+            if (signal.aborted)
+                throw failureOf(signal, null, sql.text);
+            params.forEach((v, i) => { if (v === null)
+                p.bindNull(i + 1);
+            else
+                p.bindVarchar(i + 1, v); });
+            let r;
+            try {
+                r = await p.runAndReadAll();
+            }
+            catch (e) {
+                throw failureOf(signal, e, sql.text);
+            }
+            return planNodes(JSON.parse(String(r.getRowsJS()[0]?.[1] ?? "[]")));
+        }
+        finally {
+            signal.removeEventListener("abort", interrupt);
+            p.destroySync();
+        }
+    }
+    /** each copy's rows, and the sync's facts of it (how it is kept current: src/duck/sync.ts `KeptCurrent`, never the
+     *  poll's ledger read here — a project switched to logical replication still holds its last poll there) */
+    async copies(tables, facts, filters = []) {
         const out = [];
-        for (const table of tables) {
-            const [schema, name] = table.split(".");
-            const rows = (await this.conn.runAndReadAll("SELECT estimated_size FROM duckdb_tables() WHERE database_name = current_database() AND schema_name = $1 AND table_name = $2", [schema, name])).getRowsJS()[0]?.[0];
-            const polled = polls ? (await this.conn.runAndReadAll(`SELECT strftime(polled_at AT TIME ZONE 'UTC', '%Y-%m-%d %H:%M:%S') || ' UTC' FROM ${MASON_SCHEMA}.poll_state WHERE source = $1`, [table])).getRowsJS()[0]?.[0] : undefined;
-            out.push({ table, rows: rows === undefined || rows === null ? null : Number(rows), polledAt: typeof polled === "string" ? polled : null });
+        for (const { schema, name } of tables) {
+            // the sync's source is the two names as they are; EXPLAIN says the table as a statement writes it (qualifiedName)
+            const filter = filters.find((f) => f.table === `${schema}.${name}`)?.words;
+            // a table the role filters: its counts are every row's, not the role's, and are never read or said here
+            const rows = filter !== undefined ? null : (await this.conn.runAndReadAll("SELECT estimated_size FROM duckdb_tables() WHERE database_name = current_database() AND schema_name = $1 AND table_name = $2", [schema, name])).getRowsJS()[0]?.[0];
+            out.push({ table: qualifiedName({ schema, name }), rows: rows === undefined || rows === null ? null : Number(rows), facts: facts.find((f) => f.source === `${schema}.${name}`) ?? null,
+                ...(filter !== undefined ? { filter } : {}) });
         }
         return out;
     }
     /**
-     * A CALLING STATEMENT THAT DID NOT RUN TO ITS END, WAITED FOR UNTIL ITS CALLS HAVE STOPPED, before its result is
-     * closed (`releaseAfterCleanup` in `streamed` says why). Every close is synchronous on this thread (DuckDB's node API
-     * has no other, and refuses a second query on a connection with a result open), so the close must find no task of the
-     * query inside, or waiting for, a call. The statement is marked cancelled (each call it still makes raises at once)
-     * and interrupted (its tasks stop at their next chunk), and this thread stays free — a task waiting for a call is
-     * answered within a turn of its loop — until no call of this session has been answered for `CALLS_QUIET_MS`. Never a
-     * deadline: a close while calls still come is the deadlock itself, and the calls end (each raises); a wait past
-     * `CALLS_SLOW_MS` is said once.
+     * A STATEMENT THAT DID NOT RUN TO ITS END, ENDED BY DuckDB OFF THIS THREAD before its result is closed. THIS THREAD
+     * NEVER WAITS FOR THE ENGINE'S TASKS: every close is synchronous here, a close of an open result waits for the query's
+     * tasks (Executor::CancelTasks), and this thread runs every Mason function, so a task inside a call waits for it (the
+     * suite hung for good at 0 % CPU; `tests/duck-executor.test.ts` "a vector slower than"). Waiting for quiet instead —
+     * no call for 50 ms, as this did — was no proof: a vector whose native work outlasts the window has made no call yet.
+     * And a call-free statement's tasks hold this thread as long as DuckDB's vector takes, with every session behind it.
+     *
+     * WHAT WAITS IS A FETCH (read in DuckDB 2.0's source, `ClientContext::EndQueryInternal`, and held by the test above).
+     * Every end of a query runs `CancelTasks` on the thread that ends it, and a fetch runs on libuv's pool. Interrupted
+     * first, the next fetch does not compute: it fails at once (`BufferedData::Participate`), ending the query there, its
+     * tasks waited for while this thread answers their calls; or it reads the end, or the end was read already (`r`
+     * null). So the wait is bounded by DuckDB's next chunk, on a statement that never ends by itself too, and an EMPTY chunk
+     * is the end, as `fetch` reads it: past its end a streamed result hands back empty chunks for ever, never null.
+     *
+     * THE PROBE PROVES IT (a statement performs no wait: while a result holds the connection DuckDB refuses it, "connection
+     * has an open result", before anything else). So a `SELECT 1` that ran is proof that nothing of the statement is left,
+     * and the close finds nothing to wait for. Refused, the reading above no longer holds: said as a Mason bug, and closed
+     * all the same, the watch (src/duck/executor.ts SILENT_MS) catching the deadlock if there is one. A wait past
+     * `ENDING_SLOW_MS` is said once.
      */
-    async cleanUp(seq, sql) {
-        cancelStatement(this.cancels, this.slot, seq);
+    async cleanUp(seq, sql, calls, r) {
+        if (calls)
+            cancelStatement(this.cancels, this.slot, seq);
         this.conn.interrupt();
-        const t0 = performance.now();
-        let said = false;
-        // ONE WHOLE WINDOW BEFORE THE FIRST LOOK: a call a task has queued for this thread but that has not run yet leaves
-        // `msSinceCall` at the previous call's time — long ago, when this thread was busy with another connection's close
-        // (an interrupt honoured, ~10 s for one join) — so a check made synchronously here could pass with that call still
-        // waiting, and the close deadlock on it the same way. A sleep of the window lets every queued call run first.
-        for (;;) {
-            await sleep(CALLS_QUIET_MS);
-            if (msSinceCall(this.slot) >= CALLS_QUIET_MS)
-                break;
-            if (!said && performance.now() - t0 > CALLS_SLOW_MS) {
-                said = true;
-                this.log(`a statement that did not run to its end still calls Mason's functions after ${CALLS_SLOW_MS / 1000} s; its result is closed once they stop — ${oneLine(sql)}`);
+        const slow = setTimeout(() => this.log(`a statement that did not run to its end has not stopped after ${ENDING_SLOW_MS / 1000} s; its result is closed once the engine ended it — ${oneLine(sql)}`), ENDING_SLOW_MS);
+        try {
+            if (r !== null) {
+                try {
+                    for (let c = await r.fetchChunk(); c !== null && c.rowCount > 0; c = await r.fetchChunk()) { /* its rows are nobody's */ }
+                }
+                catch (e) {
+                    // the cancel's own refusal (the interrupt, or its next call raised, 57014) is what this waits for. Anything else
+                    // is the statement's real error, met after the listener let go of its answer: no one reads it any more, and a
+                    // rethrow here would reach no caller (it would end this thread, and with it every session), so it is said
+                    if (pgErrorOf(e, sql).fields.code !== "57014")
+                        this.log(`a statement closed before its end failed while it was drained: ${e instanceof Error ? e.message : String(e)} — ${oneLine(sql)}`);
+                }
             }
+            // an abort landing now interrupts the probe past DuckDB's check, which is its proof all the same
+            try {
+                await this.conn.run("SELECT 1");
+            }
+            catch (e) {
+                if (/open result/.test(String(e)))
+                    this.log(`a statement's result is still open after the engine failed or ended it (Mason bug, report it with mason logs): closed all the same — ${oneLine(sql)}`);
+                else if (pgErrorOf(e, "SELECT 1").fields.code !== "57014")
+                    this.log(`the check that a statement ended failed: ${e instanceof Error ? e.message : String(e)} — ${oneLine(sql)}`);
+            }
+        }
+        finally {
+            clearTimeout(slow);
         }
     }
     /** Closes the connection once nothing runs on it: an answer left open drained (its statement cancelled), every
      *  answer released and every call of DuckDB settled (never under the executor: `drain`; a connection
      *  closed under a drain's pending fetch aborted the process). */
     async close() {
-        this.openAnswer?.();
+        // an answer left open is closed and drained — and one a statement still STARTING opens while this waits (the
+        // session closed between its ask and its answer: left open, the connection closed under it and its running tasks)
+        for (let last = null;;) {
+            const open = this.openAnswer;
+            if (open !== null && open !== last) {
+                last = open;
+                open();
+            }
+            await this.idle();
+            // released (null), or one already closed: done; a new one opened meanwhile: closed in turn
+            if (this.openAnswer === null || this.openAnswer === last)
+                break;
+        }
+        this.conn.closeSync();
+    }
+    /** Settles once nothing runs on the connection: every answer released and every call of DuckDB settled (an answer
+     *  closed early is answered then: src/duck/executor-worker.ts `answer.close`). */
+    async idle() {
         for (let waiting = [...this.busy, ...this.releasing]; waiting.length > 0; waiting = [...this.busy, ...this.releasing])
             await Promise.allSettled(waiting);
-        this.conn.closeSync();
     }
 }
 function exactColumns(columns, fields, exact, typeAt) {
@@ -782,7 +894,7 @@ function exactColumns(columns, fields, exact, typeAt) {
     const computed = exact.filter((e) => {
         const unreadable = readers.some((r, j) => r === null && programReads(e, j));
         if (unreadable)
-            columns[e.column] = { ...columns[e.column], divergence: `Postgres computes this numeric exactly; a part of it DuckDB computed as a double` };
+            columns[e.column] = { ...columns[e.column], divergence: `Postgres computes this numeric exactly; a part of it the engine computed as a double` };
         return !unreadable;
     });
     if (computed.length === 0)
@@ -806,7 +918,7 @@ function exactRows(computed, rows, base, sql, params) {
             }
             catch (err) {
                 if (err instanceof NumericError)
-                    throw new StatementError({ code: err.sqlstate, message: err.message }, sql);
+                    throw new StatementError({ code: err.sqlstate, message: err.message }, sql, err.gap);
                 throw err;
             }
         }
@@ -817,7 +929,7 @@ function exactRows(computed, rows, base, sql, params) {
 function computedOf(exact, fields) {
     return exact.map((e) => {
         const cells = new Set(cellsOf(e.program).filter((k) => !k.startsWith("$"))).size;
-        return `column ${e.column + 1}${fields?.[e.column] === undefined ? "" : ` "${fields[e.column].name}"`}: Postgres's exact numeric${e.text ? " as text" : ""}, from ${cells} of DuckDB's value(s)`;
+        return `column ${e.column + 1}${fields?.[e.column] === undefined ? "" : ` "${fields[e.column].name}"`}: Postgres's exact numeric${e.text ? " as text" : ""}, from ${cells} of the engine's value(s)`;
     });
 }
 /** whether the column's program reads cell j */