@mason-data/cli 0.6.2 → 0.7.0

package/dist/src/duck/kept.js

@@ -1,5 +1,9 @@
-import { MASON_SCHEMA } from "./store.js";
-import { canonicalRead, HIDDEN_PREFIX, HIDDEN_RE, hiddenName, pathKey, pathText, readOfKey, VISIBLE_CATALOG } from "./kept-route.js";
+import { DUCKDB_VERSION, MASON_SCHEMA } from "./store.js";
+import { createHash } from "node:crypto";
+import { canonicalRead, EQUAL_STEPS, HIDDEN_PREFIX, HIDDEN_RE, hiddenName, isEqualRead, isKeyRead, isOrderRead, KEY_NULL_HEX, KEY_RECOMPUTE_HEX, knownSteps, ORDER_STEPS, pathKey, pathText, readOfKey, VISIBLE_CATALOG } from "./kept-route.js";
+import { JSONB_KEY_FUNCTIONS, KEY_VERSION, libcUncovered, STORED_SUFFIX } from "./jsonb-order.js";
+import { encodeLibcOrder } from "./libc-order.js";
+import { icuNameOf } from "./rewrite.js";
 /** a path read by this many statements in the request log is stored (a larger table's also needs `KEEP_FILL_SHARE`) */
 export const KEEP_AFTER_STATEMENTS = 3;
 /** a path of a SMALL table (`SMALL_FILL_SECONDS`) read by one statement that took this long (ms, the request log's
@@ -57,6 +61,8 @@ const MEASURE_AGAIN_MS = 10 * 60_000;
 export const BACKFILL_ROWS = 50_000;
 /** how often the request log is read for new paths, and backfills continued */
 export const REVIEW_MS = 10_000;
+/** the marked rows whose documents are read to say what they are (`markedText`) */
+const MARKS_READ = 10_000;
 /** every this many reviews, each READY column's unset rows are counted (the detector) */
 const CHECK_EVERY = 6;
 export const PATHS_TABLE = `${MASON_SCHEMA}.json_paths`;
@@ -93,6 +99,50 @@ const tableOf = (source) => { const at = source.indexOf("."); return `${ident(so
 export function keptValueSql(p, value) {
     return `coalesce(__mason_json_kept(NULL, ${value}, ${literal(p.steps)}, ${p.either}, ${p.canonical}, false), ${NULL_SQL})`;
 }
+/**
+ * A JSONB'S STORED KEY, from its column's `value` (src/duck/kept-route.ts `ORDER_STEPS`): the key the statement would
+ * compute (src/duck/rewrite.ts `jsonbKeyCall`), by the same functions (src/duck/jsonb-order.ts `JSONB_KEY_FUNCTIONS`,
+ * their stored twins), never NULL — `KEY_NULL_HEX` for a NULL document (the column's NULL is a row not stored yet), and
+ * `KEY_RECOMPUTE_HEX` where the key raised, which its read computes (and raises) again.
+ */
+export function orderKeySql(c, value) {
+    const call = (name, ...args) => `${name}${STORED_SUFFIX}(NULL, ${args.join(", ")})`;
+    const key = c.kind === "bytes" ? call(JSONB_KEY_FUNCTIONS.bytes.key, value)
+        : call(JSONB_KEY_FUNCTIONS[c.kind].key, value, `list_transform(${call(JSONB_KEY_FUNCTIONS[c.kind].strings, value)}, __mason_s -> icu_sort_key(__mason_s, ${literal(icuNameOf(c))}))`);
+    return `CASE WHEN (${value}) IS NULL THEN unhex('${KEY_NULL_HEX}') ELSE coalesce(${key}, unhex('${KEY_RECOMPUTE_HEX}')) END`;
+}
+/**
+ * A STORED KEY'S ID in its `steps` (`ORDER_STEPS:<id>`): what its bytes depend on, so a key is read only where the
+ * statement's own key would be the same bytes — its collation (a libc one by its measured order's content: a glibc
+ * whose order moved is another collation), the key's encoding (`KEY_VERSION`) and the DuckDB whose ICU made its strings'
+ * keys (`DUCKDB_VERSION`: an upgrade may bring another ICU). Another id is another column, dropped at `orderBy`.
+ */
+export function orderId(c, libc, versions = `k${KEY_VERSION}:${DUCKDB_VERSION}`) {
+    if (c.kind === "bytes")
+        return `c:${versions}`;
+    if (c.kind === "icu")
+        return `icu:${c.name}:${versions}`;
+    if (libc === undefined)
+        throw new Error("a libc collation's order was not read (Mason bug: report it)");
+    return `libc:${c.icu}:${createHash("sha256").update(encodeLibcOrder(libc)).digest("hex").slice(0, 16)}:${versions}`;
+}
+/**
+ * THE C KEY'S ID, stored under a libc collation for equality (`EQUAL_STEPS:<id>`, src/duck/kept-route.ts): the C
+ * collation's order id (`orderId`), so with the key's encoding (`KEY_VERSION`) and the DuckDB (`DUCKDB_VERSION`) in it,
+ * as an order key's — a release that changes either stores it again. The libc order is not in it: the C key reads no
+ * collation, so a glibc whose order moved keeps it.
+ */
+export function equalId(versions) {
+    return versions === undefined ? orderId({ kind: "bytes" }, undefined) : orderId({ kind: "bytes" }, undefined, versions);
+}
+/** a stored column's type, and its mark of a value computed at read time, as SQL */
+const columnType = (p) => isKeyRead(p) ? "BLOB" : "VARCHAR";
+const recomputeSql = (p) => isKeyRead(p) ? `unhex('${KEY_RECOMPUTE_HEX}')` : RECOMPUTE_SQL;
+/** what a fill's measured rate is kept by (`estimate`, `lastRate`): the JSON column, and whether the fill computes a path,
+ *  a jsonb's order key or its C key — a key's function is not a path's (a list of strings, their ICU keys, the key's
+ *  bytes), nor a libc order's key the C key's (4.7 µs a row against 1.4, docs/DESIGN.md "jsonb's order"), so one's
+ *  rate is no evidence for another's */
+const measuredKey = (r) => `${r.source}\u0000${r.column}\u0000${isOrderRead(r) ? "order" : isEqualRead(r) ? "equal" : "path"}`;
 /** a read as its stored path's `->` read (src/duck/kept-route.ts pathKey): what the ledger holds */
 export const pathOf = (r) => ({ ...canonicalRead(r), asText: false });
 /** a span of seconds as a person reads it: `4 s`, `9 min 40 s`, `1 h 28 min` */
@@ -180,14 +230,73 @@ export class KeptJsonPaths {
     }
     keepNow;
     // --------------------------------------------------------------------------------------------- what a plan reads
-    ready(read) {
+    ready(asked) {
         if (!this.enabled)
             return null;
+        const read = this.own(asked);
         const p = this.paths.get(hiddenName(read));
         return p !== undefined && p.state === "ready" && pathKey(p) === pathKey(read) ? { name: p.name, exact: p.marked === 0 } : null;
     }
     visible() { return this.views; }
     generation() { return this.gen; }
+    // --------------------------------------------------------------------------------------------- a jsonb's order
+    /** the collation a jsonb's key is stored under (`orderBy`), and its id (`orderId`); null until told */
+    order = null;
+    /**
+     * A READ AS THIS KEEPER STORES IT: a jsonb's order (`ORDER_STEPS`) under the collation it was told; its equality
+     * (`EQUAL_STEPS`) as the C key (`equalId`) under a libc collation, and under C or ICU as the ORDER's read — the order's
+     * key equates there, so one column serves both and no second is ever made; any other read as it is.
+     */
+    own(r) {
+        if (isEqualRead(r) && this.order !== null && this.order.collation.kind !== "libc")
+            return { ...r, steps: `${ORDER_STEPS}:${this.order.id}` };
+        if (isEqualRead(r))
+            return { ...r, steps: `${EQUAL_STEPS}:${this.order === null ? "?" : equalId()}` };
+        return isOrderRead(r) ? { ...r, steps: `${ORDER_STEPS}:${this.order?.id ?? "?"}` } : r;
+    }
+    /** the steps a stored key would have now (`orderBy`): the order's under the collation told, the C key's under a libc
+     *  one (none under C or ICU: the order's key serves); null until told */
+    nowSteps(p) {
+        if (this.order === null)
+            return null;
+        if (isEqualRead(p))
+            return this.order.collation.kind === "libc" ? `${EQUAL_STEPS}:${equalId()}` : null;
+        return `${ORDER_STEPS}:${this.order.id}`;
+    }
+    /** a stored key of the collation statements sort and equate by now (`orderBy`): computed by writes and read by plans */
+    current(p) { const now = this.nowSteps(p); return now !== null && p.steps === now; }
+    /**
+     * THE COLLATION, TOLD (src/duck/store.ts `listener`, once the source's is known): a jsonb's stored key is that
+     * collation's. Until then a stored key is written unset and read by no plan (`open`); now the keys of another
+     * collation (the source's changed, or its glibc's order moved) are dropped, and those of this one filled where a
+     * write left them unset — the backfill's sweep — and ready again.
+     */
+    async orderBy(collation, libc) {
+        if (!this.enabled)
+            return;
+        const id = orderId(collation, libc);
+        await this.host.onWriter(async () => {
+            this.order = { id, collation, libc };
+            for (const p of [...this.paths.values()].filter((x) => isKeyRead(x) && !this.current(x) && (x.state === "backfilling" || x.state === "ready"))) {
+                const was = p.steps.slice(p.steps.indexOf(":") + 1), now = this.nowSteps(p);
+                await this.drop(this.store.writer, p, now === null
+                    ? `the database's collation is no longer a libc one (now ${id}): its order's key equates jsonb, so equality reads that one`
+                    : `stored under another collation or version (${was}, now ${now.slice(now.indexOf(":") + 1)}); stored again when its reads ask for it`);
+                this.host.log(`${p.source}: ${pathText(p)} is no longer stored: ${p.why}`);
+            }
+        });
+        void this.fills();
+    }
+    /** what a stored column takes from its JSON column's `value`: a path's `->` answer (`keptValueSql`), a jsonb's order
+     *  key (`orderKeySql` under the collation told), or its C key (`orderKeySql` under C) — unset (NULL) while the key is
+     *  not the one told (`orderBy`): the sweep fills it */
+    valueSql(p, value) {
+        if (!isKeyRead(p))
+            return keptValueSql(p, value);
+        if (!this.current(p))
+            return "NULL";
+        return orderKeySql(isEqualRead(p) ? { kind: "bytes" } : this.order.collation, value);
+    }
     /** every path with a row, for `mason status` */
     status() {
         return [...this.paths.values()].sort((a, b) => a.source.localeCompare(b.source) || pathText(a).localeCompare(pathText(b)))
@@ -198,11 +307,11 @@ export class KeptJsonPaths {
      *  write of whole rows names after the copy's own columns */
     stored(source, valueOf) {
         const kept = this.physical(source);
-        return { names: kept.map((p) => p.name), values: kept.map((p) => keptValueSql(p, valueOf(p.column))) };
+        return { names: kept.map((p) => p.name), values: kept.map((p) => this.valueSql(p, valueOf(p.column))) };
     }
     /** what an UPDATE that sets `carried` sets too: each stored column read from one of them, from its new value */
     storedSet(source, carried, valueOf) {
-        return this.physical(source).filter((p) => carried.includes(p.column)).map((p) => `${ident(p.name)} = ${keptValueSql(p, valueOf(p.column))}`);
+        return this.physical(source).filter((p) => carried.includes(p.column)).map((p) => `${ident(p.name)} = ${this.valueSql(p, valueOf(p.column))}`);
     }
     physical(source) {
         return [...this.paths.values()].filter((p) => p.source === source && (p.state === "backfilling" || p.state === "ready")).sort((a, b) => a.name.localeCompare(b.name));
@@ -230,6 +339,14 @@ export class KeptJsonPaths {
                     basis: isBasis(fillBasis) ? fillBasis : "text", rowsPerSecond: fillRate === null || fillRate === undefined ? null : Number(fillRate) },
                 filled: filledRows === null || filledRows === undefined || filledSeconds === null || filledSeconds === undefined ? null : { rows: Number(filledRows), seconds: Number(filledSeconds) } });
         }
+        // a row of a kind this Mason does not know (another release's: `knownSteps`) is dropped, its column with it (no
+        // write here could compute it, and none names it once dropped), and said with its steps: inert, never fatal
+        for (const p of this.paths.values()) {
+            if (p.state === "dropped" || knownSteps(p.steps))
+                continue;
+            await this.drop(w, p, `stored by another release as ${p.steps}, a kind this Mason does not know`);
+            this.host.log(`${p.source}: dropped the stored ${p.column} of a kind this Mason does not know (${p.steps}); its column goes at this start`);
+        }
         // a path an earlier Mason stored in another form (a column per spelling or per `->>`, its rows' marks other): its
         // name is not its path's (`hiddenName`), so it is dropped with its column, and stored again as the policy says
         const earlier = "stored by an earlier Mason in another form; stored again when its reads ask for it";
@@ -266,6 +383,14 @@ export class KeptJsonPaths {
                 continue;
             await this.drop(w, p, "its column is not in the copy");
         }
+        // a jsonb's stored key is its collation's, told after this (`orderBy`): until then a write leaves it unset and no
+        // plan reads it, so a ready one is filled again where unset first — the backfill's sweep alone (its ranges are done)
+        for (const p of this.paths.values()) {
+            if (!isKeyRead(p) || p.state !== "ready")
+                continue;
+            p.state = "backfilling";
+            this.cursors.set(p.name, Number.MAX_SAFE_INTEGER);
+        }
         await w.run(`ATTACH IF NOT EXISTS ':memory:' AS ${VISIBLE_CATALOG}`);
         for (const source of this.host.served().keys())
             await this.view(source);
@@ -286,7 +411,9 @@ export class KeptJsonPaths {
         this.reader?.closeSync();
         this.reader = null;
     }
-    /** a path's row marked dropped, with why (on the caller's connection and turn) */
+    /** a path's row marked dropped, with why (on the caller's connection and turn). Its COLUMN stays in the table until
+     *  the next start, whose reconciling of the file with the ledger (`open`, "a column with no row") drops it: every
+     *  write names only the stored columns (`stored`), and DuckDB's DROP COLUMN rewrites the table */
     async drop(c, p, why) {
         await c.run(`UPDATE ${PATHS_TABLE} SET state = 'dropped', why = ${literal(why)} WHERE name = ${literal(p.name)}`);
         if (p.state === "ready")
@@ -335,14 +462,14 @@ export class KeptJsonPaths {
             return;
         for (const p of this.physical(source).filter((x) => columns.includes(x.column))) {
             // no default: its NULL is "not stored yet", and an ADD COLUMN writes nothing into the rows (`keep`)
-            await c.run(`ALTER TABLE ${side} ADD COLUMN IF NOT EXISTS ${ident(p.name)} VARCHAR`);
+            await c.run(`ALTER TABLE ${side} ADD COLUMN IF NOT EXISTS ${ident(p.name)} ${columnType(p)}`);
             const end = Number((await c.runAndReadAll(`SELECT coalesce(max(rowid), -1) + 1 FROM ${side}`)).getRowsJS()[0][0]);
             for (let lo = 0; lo < end; lo += this.backfillRows) {
-                await run(`UPDATE ${side} SET ${ident(p.name)} = ${keptValueSql(p, ident(p.column))} WHERE rowid >= ${lo} AND rowid < ${lo + this.backfillRows} AND ${ident(p.name)} IS NULL`);
+                await run(`UPDATE ${side} SET ${ident(p.name)} = ${this.valueSql(p, ident(p.column))} WHERE rowid >= ${lo} AND rowid < ${lo + this.backfillRows} AND ${ident(p.name)} IS NULL`);
             }
             // rows the column cannot answer alone (KEPT_RECOMPUTE) turn its reads to the CASE before the side table serves;
             // counted again once it does (`swapped`)
-            const marked = Number((await c.runAndReadAll(`SELECT count(*) FROM ${side} WHERE ${ident(p.name)} = ${RECOMPUTE_SQL}`)).getRowsJS()[0][0]);
+            const marked = Number((await c.runAndReadAll(`SELECT count(*) FROM ${side} WHERE ${ident(p.name)} = ${recomputeSql(p)}`)).getRowsJS()[0][0]);
             if (marked > p.marked) {
                 if (p.marked === 0 && p.state === "ready")
                     this.gen++;
@@ -415,7 +542,7 @@ export class KeptJsonPaths {
             const read = readOfKey(key);
             if (read === null)
                 continue;
-            const path = pathOf(read), k = pathKey(path);
+            const path = this.own(pathOf(read)), k = pathKey(path);
             let e = by.get(k);
             if (e === undefined)
                 by.set(k, e = { read: path, statements: new Map() });
@@ -436,14 +563,14 @@ export class KeptJsonPaths {
     async estimate(read) {
         if (this.estimateFor !== undefined)
             return this.estimateFor(read);
-        const key = `${read.source}\u0000${read.column}`;
+        const key = measuredKey(read);
         const was = this.measured.get(key);
         if (was !== undefined && Date.now() - was.at < MEASURE_AGAIN_MS)
             return was.fill;
         this.reader ??= await this.store.instance.connect();
         const target = tableOf(read.source);
         const rows = Number((await this.reader.runAndReadAll(`SELECT count(*) FROM ${target}`)).getRowsJS()[0][0]);
-        const rate = this.lastRate(read.source, read.column, rows);
+        const rate = this.lastRate(read, rows);
         let fill;
         if (rate !== null)
             fill = measuredEstimate(rows, rate);
@@ -460,12 +587,12 @@ export class KeptJsonPaths {
         this.measured.set(key, { at: Date.now(), fill });
         return fill;
     }
-    /** the rows a second of the last fill of a JSON column (any path of it) that is evidence for a table of `rows` now
-     *  (`MEASURED_MIN_ROWS`, `MEASURED_SPAN`), or null */
-    lastRate(source, column, rows) {
+    /** the rows a second of the last fill of a JSON column (any path of it; a jsonb's key by a key's fill alone) that is
+     *  evidence for a table of `rows` now (`MEASURED_MIN_ROWS`, `MEASURED_SPAN`), or null */
+    lastRate(read, rows) {
         let last = null;
         for (const p of this.paths.values()) {
-            if (p.source !== source || p.column !== column || p.filled === null || p.filled.seconds <= 0 || p.filled.rows < MEASURED_MIN_ROWS)
+            if (measuredKey(p) !== measuredKey(read) || p.filled === null || p.filled.seconds <= 0 || p.filled.rows < MEASURED_MIN_ROWS)
                 continue;
             if (rows > MEASURED_SPAN * p.filled.rows)
                 continue;
@@ -593,11 +720,16 @@ export class KeptJsonPaths {
         const cols = this.host.served().get(read.source);
         if (cols === undefined || !this.views.has(read.source) || this.host.rebuilding(read.source))
             return false;
+        // a jsonb's key waits for its collation (`orderBy`, at the listener's start), and its C key is stored under a libc
+        // one alone (`own` folds it into the order's elsewhere): nothing to say
+        if (isKeyRead(read) && !this.current(read))
+            return false;
         const name = hiddenName(read);
         const col = cols.find((c) => c.name === read.column);
         const why = col === undefined ? `${read.column} is not a column of the copy`
             : !/^jsonb?$/.test(col.type) ? `${read.column} is a ${col.type}, not a json or jsonb`
-                : cols.some((c) => c.name.toLowerCase() === name) ? `the copy has a column named ${name} of its own` : null;
+                : isKeyRead(read) && col.type !== "jsonb" ? `${read.column} is a ${col.type}, which has no order or equality (a jsonb's is stored)`
+                    : cols.some((c) => c.name.toLowerCase() === name) ? `the copy has a column named ${name} of its own` : null;
         if (why === null)
             return true;
         const key = pathKey(read);
@@ -659,7 +791,7 @@ export class KeptJsonPaths {
             const target = tableOf(read.source);
             await w.run("BEGIN TRANSACTION");
             try {
-                await w.run(`ALTER TABLE ${target} ADD COLUMN ${ident(name)} VARCHAR`);
+                await w.run(`ALTER TABLE ${target} ADD COLUMN ${ident(name)} ${columnType(read)}`);
                 const rows = Number((await w.runAndReadAll(`SELECT count(*) FROM ${target}`)).getRowsJS()[0][0]);
                 const p = { ...pathOf(read), name, state: "backfilling", rowsTotal: rows, rowsDone: 0, statements: c.statements, slowestMs: c.slowestMs, totalMs: c.totalMs,
                     fill, filled: null, marked: 0, keptAt: new Date().toISOString(), readyAt: null, why };
@@ -682,6 +814,9 @@ export class KeptJsonPaths {
     /** a backfill continued to its end, a chunk a writer's turn (paused while its table is rebuilt: the next review goes
      *  on), then the path made ready once no row of it is unset — and read by its column alone once none is marked */
     async backfill(p) {
+        // a jsonb's key waits for its collation (`orderBy`): its rows would be filled unset
+        if (isKeyRead(p) && !this.current(p))
+            return;
         const target = tableOf(p.source);
         const end = await this.host.onWriter(async () => Number((await this.store.writer.runAndReadAll(`SELECT coalesce(max(rowid), -1) + 1 FROM ${target}`)).getRowsJS()[0][0]));
         // what this fill measures: the rows its chunks filled and the time they held the writer (never the waits for its
@@ -700,28 +835,58 @@ export class KeptJsonPaths {
             if (!done)
                 return;
         }
-        await this.host.onWriter(async () => {
-            if (this.stopped || this.host.rebuilding(p.source) || p.state !== "backfilling")
-                return;
-            // what the ranges did not reach (a row whose rowid moved), by the mark itself
-            while (await this.fillChunk(p, `rowid IN (SELECT rowid FROM ${target} WHERE ${ident(p.name)} IS NULL LIMIT ${this.backfillRows}) AND `) > 0)
-                ;
-            p.marked = await this.markedRows(p);
-            Object.assign(p, { state: "ready", readyAt: new Date().toISOString(), filled: rows > 0 && ms > 0 ? { rows, seconds: ms / 1000 } : p.filled });
-            await this.record(this.store.writer, p);
-            this.cursors.delete(p.name);
-            this.measured.delete(`${p.source}\u0000${p.column}`);
-            this.gen++;
-        });
+        // THE SWEEP: what the ranges did not reach (a row whose rowid moved, a row a write left unset while a jsonb's
+        // collation was not told), by the mark itself — a chunk a writer's turn, as the ranges are; the turn whose chunk
+        // finds none unset makes the path ready (every write computes it from then on, so none is left unset after it)
+        for (;;) {
+            const swept = await this.host.onWriter(async () => {
+                if (this.stopped || this.host.rebuilding(p.source) || p.state !== "backfilling")
+                    return "paused";
+                if (await this.fillChunk(p, `rowid IN (SELECT rowid FROM ${target} WHERE ${ident(p.name)} IS NULL LIMIT ${this.backfillRows}) AND `) > 0)
+                    return "more";
+                p.marked = await this.markedRows(p);
+                Object.assign(p, { state: "ready", readyAt: new Date().toISOString(), filled: rows > 0 && ms > 0 ? { rows, seconds: ms / 1000 } : p.filled });
+                await this.record(this.store.writer, p);
+                this.cursors.delete(p.name);
+                this.measured.delete(measuredKey(p));
+                this.gen++;
+                return "ready";
+            });
+            if (swept !== "more")
+                break;
+        }
         if (p.state === "ready") {
             const took = rows > 0 && ms > 0 ? `, ${rows.toLocaleString("en-US")} by this fill in ${spanText(ms / 1000)}${p.fill === null ? "" : `, estimated at ${spanText(p.fill.seconds)} (${p.fill.basis === "measured" ? "from the last fill" : "modelled"})`}` : "";
             this.host.log(`${p.source}: ${pathText(p)} is stored: statements read it from its column (${p.rowsDone.toLocaleString("en-US")} row(s) filled${took})` +
-                (p.marked > 0 ? `; ${p.marked.toLocaleString("en-US")} row(s) its function could not read are computed from their document when read (a Mason bug: report it, with mason logs)` : ""));
+                (p.marked > 0 ? `; ${await this.markedText(p, p.marked)}` : ""));
         }
     }
+    /**
+     * WHAT A PATH'S MARKED ROWS ARE, in words (the fill's end, the detector): a jsonb's key under a libc collation is
+     * marked where a string holds a character the measured order does not cover — the gap its sort is refused by
+     * (`libc-uncovered`), told apart by reading the marked rows' documents (`libcUncovered`, at most `MARKS_READ` of
+     * them); any other mark is a row its function could not read, a Mason bug.
+     */
+    async markedText(p, marked) {
+        let uncovered = 0, read = marked;
+        const libc = this.order?.libc;
+        if (isOrderRead(p) && this.current(p) && libc !== undefined) {
+            const docs = await this.host.onWriter(async () => (await this.store.writer.runAndReadAll(`SELECT ${ident(p.column)}::VARCHAR FROM ${tableOf(p.source)} ` +
+                `WHERE ${ident(p.name)} = ${recomputeSql(p)} LIMIT ${MARKS_READ}`)).getRowsJS());
+            read = docs.length;
+            uncovered = docs.filter(([d]) => d !== null && libcUncovered(libc, d)).length;
+        }
+        const of = read < marked ? ` (of the first ${read.toLocaleString("en-US")} read, of ${marked.toLocaleString("en-US")})` : "";
+        const said = [];
+        if (uncovered > 0)
+            said.push(`${uncovered.toLocaleString("en-US")} row(s) hold a character the database's libc order is not measured for, so a sort of them is refused by name (Mason gap libc-uncovered)`);
+        if (read > uncovered)
+            said.push(`${(read - uncovered).toLocaleString("en-US")} row(s) its function could not read are computed from their document when read (a Mason bug: report it, with mason logs)`);
+        return said.join("; ") + of;
+    }
     /** a column's rows marked to be computed at read time (KEPT_RECOMPUTE), on the caller's turn */
     async markedRows(p) {
-        return Number((await this.store.writer.runAndReadAll(`SELECT count(*) FROM ${tableOf(p.source)} WHERE ${ident(p.name)} = ${RECOMPUTE_SQL}`)).getRowsJS()[0][0]);
+        return Number((await this.store.writer.runAndReadAll(`SELECT count(*) FROM ${tableOf(p.source)} WHERE ${ident(p.name)} = ${recomputeSql(p)}`)).getRowsJS()[0][0]);
     }
     /** a READY path's marked rows counted again (its table swapped in, the detector): a change of exactness moves the
      *  plans' generation, so a plan that read the column alone is made again as the CASE, and the other way */
@@ -742,7 +907,7 @@ export class KeptJsonPaths {
         const w = this.store.writer;
         await w.run("BEGIN TRANSACTION");
         try {
-            const n = Number((await w.run(`UPDATE ${tableOf(p.source)} SET ${ident(p.name)} = ${keptValueSql(p, ident(p.column))} WHERE ${where}${ident(p.name)} IS NULL`)).rowsChanged);
+            const n = Number((await w.run(`UPDATE ${tableOf(p.source)} SET ${ident(p.name)} = ${this.valueSql(p, ident(p.column))} WHERE ${where}${ident(p.name)} IS NULL`)).rowsChanged);
             await w.run(`UPDATE ${PATHS_TABLE} SET rows_done = rows_done + ${n} WHERE name = ${literal(p.name)}`);
             await w.run("COMMIT");
             p.rowsDone += n;
@@ -756,9 +921,9 @@ export class KeptJsonPaths {
     /** THE DETECTOR: a READY column's rows still unset (a write that did not compute it: a Mason bug), filled and said;
      *  and its marked rows counted again (`settle`) */
     async check(p) {
-        await this.host.onWriter(async () => {
+        const marked = await this.host.onWriter(async () => {
             if (this.stopped || this.host.rebuilding(p.source) || p.state !== "ready")
-                return;
+                return 0;
             const unset = Number((await this.store.writer.runAndReadAll(`SELECT count(*) FROM ${tableOf(p.source)} WHERE ${ident(p.name)} IS NULL`)).getRowsJS()[0][0]);
             if (unset > 0) {
                 this.host.log(`${p.source}: ${unset} row(s) were written without the stored value of ${pathText(p)} (a Mason bug: report it, with mason logs); they are filled now`);
@@ -767,14 +932,16 @@ export class KeptJsonPaths {
             }
             const was = p.marked;
             await this.settle(p);
-            if (p.marked > was)
-                this.host.log(`${p.source}: ${p.marked} row(s) of ${pathText(p)} could not be read by its function (a Mason bug: report it, with mason logs); its reads compute them from their document`);
+            return p.marked > was ? p.marked : 0;
         });
+        // (the words read the table again on a writer's turn of their own, never inside the detector's)
+        if (marked > 0)
+            this.host.log(`${p.source}: ${pathText(p)} has marked rows, read through their document from now on: ${await this.markedText(p, marked)}`);
     }
     /** THE ORACLE'S (`eager`): every read's path stored and filled now, whatever the request log says */
     async keepAll(reads) {
         for (const r of reads) {
-            const read = pathOf(r);
+            const read = this.own(pathOf(r));
             const name = hiddenName(read);
             if (this.paths.get(name)?.state === "ready" || !this.storable(read))
                 continue;

package/dist/src/duck/pgtree.js

@@ -233,6 +233,9 @@ export function typesByLocation(tree) {
             const relation = v._node === "VAR" ? relationOf(v, rtables) : null;
             if (relation !== null)
                 typed.relation = relation;
+            const fromFunction = v._node === "VAR" ? functionColumnOf(v, rtables) : null;
+            if (fromFunction !== null)
+                typed.functionColumn = fromFunction;
             const list = out.get(loc);
             if (list)
                 list.push(typed);
@@ -255,6 +258,17 @@ function relationOf(v, rtables) {
     const relid = rte === undefined ? null : num(rte.relid ?? null);
     return rte === undefined || attno === null || attno < 1 || num(rte.rtekind ?? null) !== RTE_RELATION || relid === null || relid === 0 ? null : { relid, attno };
 }
+const RTE_FUNCTION = 3;
+/** Typed.functionColumn */
+function functionColumnOf(v, rtables) {
+    const up = num(v.varlevelsup ?? null) ?? 0, varno = num(v.varno ?? null), attno = num(v.varattno ?? null);
+    const rte = varno === null ? undefined : rtables[rtables.length - 1 - up]?.[varno - 1];
+    if (rte === undefined || attno === null || attno < 1 || num(rte.rtekind ?? null) !== RTE_FUNCTION || !isNode(rte.eref ?? null))
+        return null;
+    const eref = rte.eref;
+    const column = strings(eref.colnames)[attno - 1];
+    return typeof eref.aliasname === "string" && column !== undefined ? { alias: eref.aliasname, column } : null;
+}
 const strings = (v) => Array.isArray(v) ? v.map((x) => typeof x === "string" ? x : "") : [];
 /** Typed.qualifier: the name of a VAR's FROM item when another FROM item of its query has the column in another case,
  *  or when the VAR is an outer query's (a correlated subquery's bare `i4`, which DuckDB binds to the subquery's own
@@ -327,6 +341,32 @@ export function caseTwins(tree) {
     visit(tree);
     return out;
 }
+/**
+ * Whether a JOIN … USING or a NATURAL JOIN of the analysed tree joins on a column of one of `types` (jsonb, jsonb[]):
+ * Postgres equates the columns by that type's `=`, which DuckDB's USING would not (it compares the text). The join's
+ * condition is Postgres's own (its `quals`), at no offset of the statement, so it is read here, not by the rewrite.
+ */
+export function joinsUsing(tree, types) {
+    const compares = (v) => {
+        if (Array.isArray(v))
+            return v.some(compares);
+        if (v === undefined || !isNode(v))
+            return false;
+        if (v._node === "OPEXPR" && Array.isArray(v.args) && v.args.some((a) => isNode(a) && types.has(exprType(a)?.type ?? 0)))
+            return true;
+        return Object.entries(v).some(([k, x]) => k !== "_node" && compares(x));
+    };
+    const visit = (v) => {
+        if (Array.isArray(v))
+            return v.some(visit);
+        if (v === undefined || !isNode(v))
+            return false;
+        if (v._node === "JOINEXPR" && Array.isArray(v.usingClause) && v.usingClause.length > 0 && compares(v.quals))
+            return true;
+        return Object.entries(v).some(([k, x]) => k !== "_node" && visit(x));
+    };
+    return visit(tree);
+}
 /** The analysed query's output columns, in order (resjunk ones, which a sort may add, left out): their types. */
 export function outputTypes(tree) {
     if (!isNode(tree) || tree._node !== "QUERY" || !Array.isArray(tree.targetList))

package/dist/src/duck/plan.js

@@ -16,6 +16,7 @@ import { selectListAfterLimit } from "./late.js";
 import { pgTyplen } from "./types.js";
 import { datumText, masonCalls, rewrite, treeDifference, Unrewritable } from "./rewrite.js";
 import { exposes, hideKept, readKey, searchSchemas } from "./kept-route.js";
+import { distinctCountTables, distinctKeyCounts, uniqueColumnsOf, uniqueColumnsSql } from "./distinct-key.js";
 const JSON_OID = 114, JSONB_OID = 3802;
 /** what the rewrite asks of the stored JSON paths (src/duck/kept-route.ts KeptLookup): the copied table and column of
  *  each json or jsonb column the statement reads from a table (Typed.relation), by the catalog's OIDs; null when it
@@ -33,6 +34,15 @@ async function keptLookup(catalog, types, kept) {
     const at = new Map(rows.map(([relid, attno, schema, table, column]) => [`${relid}.${attno}`, { source: `${schema}.${table}`, column: column }]));
     return { at: (relid, attno) => at.get(`${relid}.${attno}`) ?? null, ready: (read) => kept.ready(read) };
 }
+/** the count(DISTINCT x) of the statement that are count(x) (src/duck/distinct-key.ts), by their offsets into the
+ *  analysed text: x unique by the CATALOG's own pg_index — the source's keys as the catalog declares them — read only
+ *  for a statement that counts DISTINCT values at all */
+async function distinctKeys(catalog, tree) {
+    const relids = distinctCountTables(tree);
+    if (relids.length === 0)
+        return new Set();
+    return distinctKeyCounts(tree, uniqueColumnsOf(await catalogRows(catalog, uniqueColumnsSql(relids))));
+}
 /**
  * DuckDB's keywords that Postgres's are not: a column named `at` (DuckDB's AT TIME ZONE) or `qualify` is an identifier
  * the deparser writes bare, as Postgres reads it, and DuckDB refuses as a syntax error. The deparser's one quoting
@@ -119,7 +129,9 @@ export function missingRelationHint(relation, copies) {
         case "copied": return `${c.source} was just copied and joins the catalog as it is rebuilt: run the statement again in a moment (if it stays missing, mason logs says why)`;
     }
 }
-export async function planDuck(catalog, text, stmt, base, paramOids, searchPath, timeZone, collation, verified, userTypes = new Map(), copies = () => [], depth = 0, kept = null) {
+/** A statement's plan. `wireBound`: its rows are bounded by a row count an Execute sets on the wire, which its SQL
+ *  does not say (src/duck/rewrite.ts markBounds; the listener asks for it where PlannedSql.unboundedLateral says so) */
+export async function planDuck(catalog, text, stmt, base, paramOids, searchPath, timeZone, collation, verified, userTypes = new Map(), copies = () => [], depth = 0, kept = null, wireBound = false) {
     const started = performance.now();
     const analysis = await analyze(catalog, text, paramOids, pathSql(searchPath), timeZone);
     if ("refused" in analysis) {
@@ -138,7 +150,7 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
         if (depth >= MAX_VIEW_DEPTH)
             throw viewRefused(`Mason cannot yet answer views read through more than ${MAX_VIEW_DEPTH} other views`, "read the tables the views read");
         try {
-            return await planDuck(catalog, expanded.text, expanded.stmt, 0, analysis.params, searchPath, timeZone, collation, verified, userTypes, copies, depth + 1, kept);
+            return await planDuck(catalog, expanded.text, expanded.stmt, 0, analysis.params, searchPath, timeZone, collation, verified, userTypes, copies, depth + 1, kept, wireBound);
         }
         catch (e) {
             if (!(e instanceof StatementError))
@@ -183,24 +195,25 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
                 f.oid = source;
         }
     }
-    let tree, dropLast, exact, clock, jsonReads;
+    let tree, dropLast, exact, clock, jsonReads, unboundedLateral;
     // a table's row the source has otherwise (copied in part: no source OID for it) is not the source's row
     const sourceRows = new Map([...rowTypes].filter(([oid]) => userTypes.has(catalogTypes.get(oid) ?? "")));
     // the stored JSON paths the statement's JSON reads may be answered from (src/duck/kept-route.ts)
     const lookup = kept === null ? null : await keptLookup(catalog, analysis.types, kept);
+    const keyCounts = await distinctKeys(catalog, analysis.tree);
     const analysed = performance.now();
-    const rewritten = (l = lookup) => rewrite(stmt, base, analysis, collation, catalogTypes, sourceRows, timeZone, enums, l);
+    const rewritten = (l = lookup) => rewrite(stmt, base, analysis, collation, catalogTypes, sourceRows, timeZone, enums, l, wireBound, keyCounts);
     try {
-        ({ tree, dropLast, exact, clock, jsonReads } = rewritten());
+        ({ tree, dropLast, exact, clock, jsonReads, unboundedLateral } = rewritten());
         // the oracle's (`eager`): every read stored before the plan is made, and the plan made again over them
         if (kept?.keepNow != null && lookup !== null && jsonReads.some((r) => kept.ready(r) === null)) {
             await kept.keepNow(jsonReads);
-            ({ tree, dropLast, exact, clock, jsonReads } = rewritten());
+            ({ tree, dropLast, exact, clock, jsonReads, unboundedLateral } = rewritten());
         }
         // a statement that lists a copy's columns without naming them reads none from a stored column (hideKept): made
         // again with every read its call, the reads still recorded
         if (lookup !== null && exposes(tree))
-            ({ tree, dropLast, exact, clock, jsonReads } = rewritten({ at: (relid, attno) => lookup.at(relid, attno), ready: () => null }));
+            ({ tree, dropLast, exact, clock, jsonReads, unboundedLateral } = rewritten({ at: (relid, attno) => lookup.at(relid, attno), ready: () => null }));
     }
     catch (e) {
         // a gap says its id (docs/SQL-SUPPORT.md); Postgres's own refusal is its SQLSTATE, and no gap
@@ -240,7 +253,7 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
         throw new StatementError({ code: "0A000", message: "Mason cannot run this statement on DuckDB exactly: its rewritten form does not read back as itself (a Mason gap: report it, with the statement)",
             detail: `read back: ${difference}; DuckDB SQL: ${sql}` }, text);
     }
-    const planned = { text: sql, functions: masonCalls(tree) };
+    const planned = { text: sql, functions: masonCalls(tree), ...(unboundedLateral ? { unboundedLateral: true } : {}) };
     return { sql: planned, params: analysis.params, fields: fields?.map((f) => ({ ...f, size: pgTyplen(f.oid) })) ?? null, dropLast, ...(exact.length > 0 ? { exact } : {}), resolved: keys,
         ...(Object.keys(clock).length > 0 ? { clock } : {}), ...(jsonReads.length > 0 ? { jsonReads: jsonReads.map(readKey) } : {}), timing: { analysisMs: analysed - started, rewriteMs: performance.now() - analysed } };
 }