@mason-data/cli 0.6.2 → 0.7.0

package/dist/src/duck/functions.js

@@ -10,13 +10,15 @@
  *    binds a lambda MACRO's constant arguments wrongly once its postgres extension is loaded: no macro is used).
  * Every name is `__mason_*`, which no Postgres function has.
  */
-import { BIGINT, BOOLEAN, DATE, DOUBLE, DuckDBDateValue, DuckDBIntervalValue, DuckDBScalarFunction, DuckDBTimestampTZValue, DuckDBTimestampValue, INTEGER, INTERVAL, LIST, TIMESTAMP, TIMESTAMPTZ, VARCHAR, listValue } from "@duckdb/node-api";
+import { BIGINT, BLOB, BOOLEAN, DATE, DOUBLE, DuckDBBlobValue, DuckDBDateValue, DuckDBIntervalValue, DuckDBScalarFunction, DuckDBTimestampTZValue, DuckDBTimestampValue, INTEGER, INTERVAL, LIST, TIMESTAMP, TIMESTAMPTZ, VARCHAR, listValue } from "@duckdb/node-api";
 import { pgEvalSync, PgEvalError } from "./pg-eval.js";
+import { MARKED_FUNCTIONS } from "./kept-route.js";
 import { libcKept, libcKey } from "./libc-order.js";
 import { formatFloat } from "../api/postgres/encode.js";
 import { floatAsNumeric } from "./types.js";
 import { intervalText } from "./types.js";
 import { jsonbText, parseJson, writeJson, quoteJson } from "./jsonb.js";
+import { JSONB_KEY_FUNCTIONS, jsonbKey, jsonbLibcKept, jsonbStrings, STORED_SUFFIX } from "./jsonb-order.js";
 import { compare, div, evaluateText, NumericError, numericText, parseNumeric } from "../lib/pg-numeric.js";
 import { toCharNumber, toCharTime } from "./to-char.js";
 import { atTimeZoneOf } from "../lib/timezone.js";
@@ -445,7 +447,12 @@ function contains(a, b) {
 }
 /** `a @> b` on jsonb (jsonb_contains): their roots must both be objects or both not, then JsonbDeepContains */
 export function jsonbContains(a, b) {
-    const x = root(parseJson(a)), y = root(parseJson(b));
+    const pa = parseJson(a), pb = parseJson(b);
+    const x = root(pa), y = root(pb);
+    // a raw scalar (stored as a one-element array) contains no array: `'1' @> '[1]'` is false, `'[1]' @> '1'` true
+    // (JsonbDeepContains refuses a raw scalar on the left of a real array)
+    if (pa.t !== "arr" && pa.t !== "obj" && pb.t === "arr")
+        return false;
     return (x.t === "obj") === (y.t === "obj") && contains(x, y);
 }
 /** `j ? key` on jsonb (jsonb_exists): an object's top-level key, or a string element of an array (a string scalar's own) */
@@ -482,6 +489,7 @@ function scalar(conn, name, params, returns, body, opts = {}) {
         specialHandling: true,
         mainFunction: (info, input, out) => {
             FUNCTION_CALLS.set(name, (FUNCTION_CALLS.get(name) ?? 0) + 1);
+            FUNCTION_ROWS.set(name, (FUNCTION_ROWS.get(name) ?? 0) + input.rowCount);
             const all = Array.from({ length: input.columnCount }, (_, c) => input.getColumnVector(c));
             if (cancelled(info, all, input.rowCount))
                 return;
@@ -598,6 +606,33 @@ const steps = (json, either) => JSON.parse(json)
 export function registerLibcOrder(conn, order) {
     scalar(conn, "__mason_libc_kept", [VARCHAR], VARCHAR, ([s]) => libcKept(order, String(s)));
     scalar(conn, "__mason_libc_key", [VARCHAR, VARCHAR], VARCHAR, ([s, icu]) => libcKey(order, String(s), String(icu)));
+    // jsonb's order under it (src/duck/jsonb-order.ts): the characters of each string ICU's levels read, then the key
+    // from ICU's keys of them (src/duck/rewrite.ts jsonbKey hands them in)
+    keyScalar(conn, JSONB_KEY_FUNCTIONS.libc.strings, [VARCHAR], LIST(VARCHAR), ([j]) => listValue(jsonbLibcKept(order, String(j))));
+    keyScalar(conn, JSONB_KEY_FUNCTIONS.libc.key, [VARCHAR, LIST(VARCHAR)], BLOB, ([j, keys]) => new DuckDBBlobValue(jsonbKey(String(j), { kind: "libc", order, icuKeys: icuKeysOf(keys) })));
+}
+/**
+ * A FUNCTION OF A JSONB'S KEY (src/duck/jsonb-order.ts `JSONB_KEY_FUNCTIONS`), and its stored twin (`STORED_SUFFIX`),
+ * which a write of the copy computes a stored key with (src/duck/kept.ts `orderKeySql`): the same body, answering NULL
+ * where it raises — a write of the copy (the stream's, the poll's) must not fail over one row's document. The stored
+ * key's SQL marks that row to be computed when read (src/duck/kept-route.ts `KEY_RECOMPUTE_HEX`), so the read raises its own error
+ * (a libc string the order does not cover: `libc-uncovered`), as the statement would without the stored key.
+ */
+function keyScalar(conn, name, params, returns, body) {
+    scalar(conn, name, params, returns, body);
+    scalar(conn, `${name}${STORED_SUFFIX}`, params, returns, (args) => {
+        try {
+            return body(args);
+        }
+        catch {
+            return null;
+        } // the row's mark: its read computes it, and raises (above)
+    });
+}
+/** the ICU keys DuckDB made of a jsonb's strings (a NULL among them is a Mason bug: every string has one) */
+function icuKeysOf(v) {
+    return itemsOf(v).map((k) => { if (k === null)
+        throw new Error("an ICU key of a jsonb's string is NULL: a Mason bug"); return k; });
 }
 /**
  * EVERY CALL CARRIES ITS SESSION, FIRST (the test server: a client leaving mid-statement hung the process
@@ -618,6 +653,9 @@ export const CANCELLED = "mason[57014]: canceling statement due to user request"
 /** how many vectors each function was called with (a call is a hop to the main thread): what a test or a measurement
  *  reads to know whether a statement reached a function at all (the rewrite answers a `->>` natively where it can) */
 export const FUNCTION_CALLS = new Map();
+/** how many rows each function was handed, across its calls: what a plan's cost in function calls is measured by,
+ *  whatever the chunking (tests/duck-plans.test.ts) */
+export const FUNCTION_ROWS = new Map();
 /** the cancel table the calls read: the executor's, shared with the listener (`useCancelTable`), else one of this thread's own */
 let cancels = cancelTable();
 export function useCancelTable(t) { cancels = t; }
@@ -647,10 +685,27 @@ export function registerFunctions(conn) {
     scalar(conn, "__mason_json_get", [VARCHAR, VARCHAR, BOOLEAN, BOOLEAN, BOOLEAN], VARCHAR, ([j, s, either, canonical, asText]) => jsonGet(String(j), steps(String(s), either === true), canonical === true, asText === true));
     // the same, as a stored JSON path keeps it (keptJsonValue, src/duck/kept.ts): what the copy's writes compute
     scalar(conn, "__mason_json_kept", [VARCHAR, VARCHAR, BOOLEAN, BOOLEAN, BOOLEAN], VARCHAR, ([j, s, either, canonical, asText]) => keptJsonValue(String(j), steps(String(s), either === true), canonical === true, asText === true));
-    scalar(conn, "__mason_json_elements", [VARCHAR, BOOLEAN, BOOLEAN], LIST(VARCHAR), ([j, canonical, asText]) => listValue(jsonElements(String(j), canonical === true, asText === true)));
-    scalar(conn, "__mason_json_keys", [VARCHAR, BOOLEAN], LIST(VARCHAR), ([j, canonical]) => listValue(jsonMembers(String(j), canonical === true, false).map(([k]) => k)));
-    scalar(conn, "__mason_json_values", [VARCHAR, BOOLEAN, BOOLEAN], LIST(VARCHAR), ([j, canonical, asText]) => listValue(jsonMembers(String(j), canonical === true, asText === true).map(([, v]) => v)));
+    // a document's rows as a list each (elements, keys, values); `orNull` (the last argument): never empty, a NULL or
+    // empty document's list one NULL — a LEFT JOIN LATERAL … ON TRUE's one row of NULLs, as an INNER lateral DuckDB runs
+    // in its rows' order and stops at a LIMIT (src/duck/rewrite.ts `leftLateral`)
+    const rows = (j, orNull, list) => {
+        const items = j === null || j === undefined ? null : list(String(j));
+        return orNull === true && (items === null || items.length === 0) ? listValue([null]) : items === null ? null : listValue(items);
+    };
+    scalar(conn, "__mason_json_elements", [VARCHAR, BOOLEAN, BOOLEAN, BOOLEAN], LIST(VARCHAR), ([j, canonical, asText, orNull]) => rows(j, orNull, (t) => jsonElements(t, canonical === true, asText === true)), { nulls: "pass" });
+    scalar(conn, "__mason_json_keys", [VARCHAR, BOOLEAN, BOOLEAN], LIST(VARCHAR), ([j, canonical, orNull]) => rows(j, orNull, (t) => jsonMembers(t, canonical === true, false).map(([k]) => k)), { nulls: "pass" });
+    scalar(conn, "__mason_json_values", [VARCHAR, BOOLEAN, BOOLEAN, BOOLEAN], LIST(VARCHAR), ([j, canonical, asText, orNull]) => rows(j, orNull, (t) => jsonMembers(t, canonical === true, asText === true).map(([, v]) => v)), { nulls: "pass" });
     scalar(conn, "__mason_jsonb_text", [VARCHAR], VARCHAR, ([j]) => jsonbText(String(j)));
+    // jsonb's order and equality as bytes (src/duck/jsonb-order.ts): under C, and under ICU from ICU's keys of the
+    // document's strings, which the rewrite has DuckDB make of `__mason_jsonb_strings` (src/duck/rewrite.ts jsonbKey)
+    keyScalar(conn, JSONB_KEY_FUNCTIONS.bytes.key, [VARCHAR], BLOB, ([j]) => new DuckDBBlobValue(jsonbKey(String(j), { kind: "bytes" })));
+    // a stored column's marked row read exact raises its error (src/duck/kept-route.ts markedError): never DuckDB's
+    // volatile error(), which a window's ORDER BY refuses
+    const marked = ([, code, message]) => { throw new PgEvalError(String(code), String(message)); };
+    scalar(conn, MARKED_FUNCTIONS.key, [VARCHAR, VARCHAR, VARCHAR], BLOB, marked);
+    scalar(conn, MARKED_FUNCTIONS.value, [VARCHAR, VARCHAR, VARCHAR], VARCHAR, marked);
+    keyScalar(conn, JSONB_KEY_FUNCTIONS.icu.strings, [VARCHAR], LIST(VARCHAR), ([j]) => listValue(jsonbStrings(String(j))));
+    keyScalar(conn, JSONB_KEY_FUNCTIONS.icu.key, [VARCHAR, LIST(VARCHAR)], BLOB, ([j, keys]) => new DuckDBBlobValue(jsonbKey(String(j), { kind: "icu", icuKeys: icuKeysOf(keys) })));
     // jsonb's @>, ?, ?| and ?& (DuckDB's json_contains and json_exists answer otherwise on arrays), json(b)_array_length
     scalar(conn, "__mason_jsonb_contains", [VARCHAR, VARCHAR], BOOLEAN, ([a, b]) => jsonbContains(String(a), String(b)));
     scalar(conn, "__mason_jsonb_exists", [VARCHAR, VARCHAR], BOOLEAN, ([j, k]) => jsonbExists(String(j), String(k)));

package/dist/src/duck/gaps.js

@@ -33,11 +33,12 @@ export const GAPS = {
     "json-of-type": { kind: "refused", what: "to_json / array_to_json / json_build_object of a value (or an array of elements) of a type Mason does not write as json", postgres: "answers", mason: "refused", wayOut: "cast the value to text, a number, a boolean, a timestamp or json first" },
     "extract-julian": { kind: "refused", what: "extract(julian FROM …)", postgres: "answers", mason: "refused", wayOut: "extract(epoch FROM …) / 86400 + 2440587.5" },
     "with-ordinality": { kind: "refused", what: "a function in FROM … WITH ORDINALITY", postgres: "numbers the rows", mason: "refused", wayOut: "row_number() OVER () instead" },
+    "array-subquery-element": { kind: "refused", what: "ARRAY(SELECT x …) whose x is a numeric, an array or a row; x = ANY or ALL (ARRAY(SELECT …))", postgres: "a numeric[] computed exactly, a multi-dimensional array (2202E on an empty element, 22004 on a NULL one), a record[]", mason: "refused (DuckDB makes a list of doubles, of lists, of structs; it binds ANY (ARRAY(SELECT …)) as a comparison of its own)", wayOut: "ARRAY(SELECT x::text …), array_agg(x ORDER BY …) in a subquery, or x = ANY (SELECT …) without the ARRAY" },
     "multidim-array": { kind: "refused", what: "a multi-dimensional array literal", postgres: "answers", mason: "refused", wayOut: "a one-dimensional array" },
     "format-specifiers": { kind: "refused", what: "format() with a specifier other than %s (%I, %L, widths, positions)", postgres: "answers", mason: "refused", wayOut: "concatenate with ||, or %s only" },
     "to-char-patterns": { kind: "refused", what: "to_char of an interval or a time; its TZ, SP, EEEE, RN, V, B, L patterns; a day or month name padded to 9 characters in a parse", postgres: "answers", mason: "refused", wayOut: "the patterns listed in the hint; extract(… FROM interval) for an interval's parts" },
     "to-timestamp-format": { kind: "refused", what: "to_timestamp / to_date with a format that is not a literal", postgres: "answers", mason: "refused", wayOut: "write the format as a string literal" },
-    "libc-uncovered": { kind: "refused", what: "ORDER BY, <, >, min / max of text holding a combining mark (e + U+0301), a Thai or Lao vowel written before its consonant, a private-use or unassigned code point, under a libc collation", postgres: "answers, in glibc's order", mason: "refused when a value holding one is ordered (the order is read from the database, and glibc orders these otherwise than Mason can follow)", wayOut: "normalize the text (normalize(x, NFC)), or ORDER BY x COLLATE \"C\" for byte order" },
+    "libc-uncovered": { kind: "refused", what: "ORDER BY, <, >, min / max of text (or of a jsonb, by its strings) holding a combining mark (e + U+0301), a Thai or Lao vowel written before its consonant, a private-use or unassigned code point, under a libc collation", postgres: "answers, in glibc's order", mason: "refused when a value holding one is ordered (the order is read from the database, and glibc orders these otherwise than Mason can follow)", wayOut: "normalize the text (normalize(x, NFC)), or ORDER BY x COLLATE \"C\" for byte order" },
     "libc-symbol-punctuation": { kind: "differs", what: "text ordered under a libc collation where punctuation or a space follows a digit or a symbol and a later letter differs only in case ('1-X' beside '1x')", postgres: "the punctuation outranks the case: 1-X < 1x", mason: "the case outranks the punctuation: 1x < 1-X", wayOut: "ORDER BY lower(x), x when the case must not decide first" },
     "percentile-text-libc": { kind: "refused", what: "percentile_disc / mode() WITHIN GROUP (ORDER BY text) under a libc collation", postgres: "answers", mason: "refused", wayOut: "order by a number, or ORDER BY … LIMIT in a subquery" },
     "infinite-extract": { kind: "refused", what: "extract(year, decade, century, millennium, isoyear or epoch FROM) an infinite date or timestamp", postgres: "Infinity (a numeric)", mason: "refused when a row holds one (DuckDB's DECIMAL has no Infinity)", wayOut: "filter the infinite values out (isfinite(x)), or date_part(…), which is a double" },
@@ -45,7 +46,7 @@ export const GAPS = {
     "quote-ident": { kind: "refused", what: "quote_ident()", postgres: "quotes an identifier when it must be", mason: "refused (it needs Postgres's keyword list)", wayOut: "quote it in the statement, or format('%s', …)" },
     "div-numeric": { kind: "refused", what: "div() of a value that is not an integer", postgres: "the integer quotient of two numerics", mason: "refused", wayOut: "trunc(a / b)" },
     "jsonb-set": { kind: "refused", what: "jsonb_set, jsonb_insert, the #- operator, jsonb - of a text[] or an index", postgres: "answers", mason: "refused", wayOut: "build the value with jsonb_build_object" },
-    "jsonb-compare": { kind: "refused", what: "ORDER BY, GROUP BY, DISTINCT or <, >, = over jsonb (Postgres orders and equates jsonb by its own rules: numbers by value, objects by size first)", postgres: "answers", mason: "refused", wayOut: "compare a member: j ->> 'k', or j::text for the canonical text" },
+    "jsonb-compare": { kind: "refused", what: "a jsonb compared by GREATEST / LEAST, WITHIN GROUP (ORDER BY …), IN or = ANY of a subquery or of an array not written out, BETWEEN SYMMETRIC, JOIN … USING or NATURAL JOIN, UNION / INTERSECT / EXCEPT (but UNION ALL), SELECT DISTINCT *, GROUPING SETS / ROLLUP / CUBE, or an aggregate's DISTINCT but count's; ORDER BY the number of a jsonb column a * lists; a row holding a jsonb compared, grouped or sorted; a jsonb[] sorted, grouped, compared, contained (@>, <@, &&) or searched (array_position, array_remove)", postgres: "answers, by jsonb's own order and equality", mason: "refused (ORDER BY, GROUP BY, DISTINCT [ON], =, <, >, IN (…), = ANY (ARRAY[…]), BETWEEN, IS DISTINCT FROM, NULLIF, CASE x WHEN, PARTITION BY and count(DISTINCT …) of a jsonb are answered in its order)", wayOut: "compare the jsonb in one of the answered forms, a member (j ->> 'k'), or j::text for the canonical text" },
     "text-array-order": { kind: "refused", what: "ORDER BY, GROUP BY, DISTINCT, min, max or <, > over a text[] under the database's collation", postgres: "compares the elements under the collation", mason: "refused", wayOut: "sort by an element (a[1]), or by array_to_string(a, ',')" },
     "sum-real": { kind: "refused", what: "sum() of a real (float4), as an aggregate or a window", postgres: "accumulates in single precision", mason: "refused (DuckDB accumulates in double precision)", wayOut: "sum(x::double precision)" },
     "interval-aggregates": { kind: "refused", what: "sum() or avg() of an interval", postgres: "adds months, days and time apart", mason: "refused", wayOut: "sum(extract(epoch FROM x)) for the seconds" },
@@ -65,14 +66,18 @@ export const GAPS = {
     "infinite-interval": { kind: "refused", what: "an infinite interval read from a text (Postgres 17+)", postgres: "infinity, -infinity", mason: "refused when a row holds one (DuckDB's interval has no infinity)", wayOut: "filter it out first, or read it as text" },
     "relative-time-literal": { kind: "refused", what: "a relative time combined with anything else ('today 10:00', 'now'::date + …  written as one literal), or read from a row as a date or a time", postgres: "the transaction's start (a row's value: when the row is read)", mason: "refused (a bare 'now', 'today', 'tomorrow', 'yesterday' literal is answered: the transaction's start at the Parse)", wayOut: "current_date + time '10:00', now(), current_date + 1" },
     // ---------------------------------------------------------------------------------------------- differs
-    "limit-errors": { kind: "differs", what: "a select-list value that raises (`'x'::int`, a bad JSON number) in a row a LIMIT leaves out, where no index gives the ORDER BY", postgres: "raises: its plan sorts every row, its select list computed for each (through an index it stops at the limit and answers)", mason: "answers: a sorted, limited select list is computed for the rows the limit keeps (src/duck/late.ts), as Postgres's index scan does", wayOut: "none needed for an answer; filter the bad rows out to have neither raise" },
+    "limit-errors": { kind: "differs", what: "a value that raises (`'x'::int`, a bad JSON number) in a row a LIMIT leaves out: in the select list, where no index gives the ORDER BY; or in the argument of a LEFT JOIN of a list's rows ON TRUE (unnest(ARRAY[…, 'x'::int]), jsonb_each of a non-object), under a LIMIT with no ORDER BY", postgres: "the select list's raises: its plan sorts every row, its select list computed for each (through an index it stops at the limit and answers); the lateral's answers: it stops at the limit", mason: "the select list's answers: a sorted, limited select list is computed for the rows the limit keeps (src/duck/late.ts), as Postgres's index scan does; the lateral's raises: DuckDB computes its argument for a whole vector of rows (2,048) before the limit stops it", wayOut: "none needed for the select list's answer; filter the bad rows out to have neither raise" },
+    "limit-lateral-filter": { kind: "differs", what: "a LEFT JOIN of a json document's rows ON TRUE (json_each, jsonb_array_elements, …) under a LIMIT, or a row count an Execute sets on the wire, and no ORDER BY, with a filter above the join that DuckDB does not push down to the base: a WHERE on the lateral's own column (d.key <> 'n'), one holding a bound parameter ($1) in a SELECT around the join, or one in a SELECT that reads the join through a CTE", postgres: "the first rows of the base, in its order, each row's members in theirs", mason: "DuckDB plans the lateral under such a filter as a delimiter join and hands the rows back in an order of its own, so the LIMIT keeps other rows (a filter on the base in the join's own SELECT, literal or parameter, or a literal one around it through a derived table, keeps Postgres's)", wayOut: "ORDER BY the base's key before the LIMIT, or filter the base in the SELECT that holds the join", heldBy: "tests/duck-wire.test.ts" },
+    "limit-lateral-argument": { kind: "differs", what: "a LEFT JOIN of a list's rows ON TRUE (unnest, json_each, jsonb_array_elements, …) under a LIMIT, or a row count an Execute sets on the wire, and no ORDER BY, whose argument holds a subquery (unnest(ARRAY(SELECT …))), or reads two or more columns of the row and no array (CASE WHEN t.kind = 'doc' THEN t.body END) in a SELECT whose list is a bare * or inside a join with a name of its own ((a LEFT JOIN … ON TRUE) AS j)", postgres: "the first rows of the base, in its order, each row's members in theirs", mason: "DuckDB plans such a lateral as a delimiter join and hands the rows back in an order of its own, so the LIMIT keeps other rows (elsewhere an argument of two columns is computed in a step of its own before the lateral, which a * would read as a column; a subquery is a delimiter join in every form)", wayOut: "name the columns instead of *, or ORDER BY the base's key before the LIMIT" },
+    "window-partition-order": { kind: "differs", what: "a window function that reads its frame in order (row_number, lag, lead, first_value, last_value, nth_value, ntile) over a window with PARTITION BY and no ORDER BY", postgres: "a partition's rows in the order its sort hands them back: in practice the order they came in", mason: "a partition's rows in the order DuckDB's partitioning hands them back, another one", wayOut: "ORDER BY in the window (OVER (PARTITION BY a ORDER BY b))" },
     "mixed-scale-numeric": { kind: "differs", what: "a numeric column whose values have several scales: an integer UNION ALL a numeric, lead(x, 1, 0) beside numeric(2,1) values, an unconstrained column holding 1 and 1.50", postgres: "each value its own scale: 1, 1.5", mason: "one scale per column: 1.0, 1.5", wayOut: "cast to numeric(p, s), or round(x, s)" },
     "unicode-16-case": { kind: "differs", what: "upper / lower of letters added in Unicode 16", postgres: "maps them (its own Unicode 16 tables)", mason: "leaves them as they are (DuckDB's ICU predates Unicode 16)", wayOut: "none needed for letters before Unicode 16" },
-    "float-last-digit": { kind: "differs", what: "stddev / variance of a float, and ln, log, exp, power of a double: the last digit", postgres: "its algorithm and libm's", mason: "DuckDB's: equal to about 1e-15", wayOut: "round(x::numeric, 12)" },
+    "float-last-digit": { kind: "differs", what: "stddev / variance of a float, and ln, log, exp, power of a double: the last digit may differ", postgres: "its algorithm and its machine's libm", mason: "DuckDB's and its machine's: alike on some pairs of machines, apart in the 16th–17th digit on others", wayOut: "round(x::numeric, 12), or compare within a tolerance (abs(a - b) <= 1e-12 * abs(b))" },
     "negative-zero": { kind: "differs", what: "-0 beside 0 in greatest / least / percentile_disc", postgres: "keeps the one it met first", mason: "may answer the other", wayOut: "x + 0 makes -0 into 0" },
     "float-overflow": { kind: "differs", what: "exp / power of a double past its range, or so small it underflows", postgres: "an error (22003)", mason: "Infinity, or 0", wayOut: "bound the argument" },
     "min-max-scale-tie": { kind: "differs", what: "min / max of a numeric expression (a quotient) where equal values have different scales (0 / 7 and 0 / 1000000)", postgres: "the last of them it reads, so the scan order decides the trailing zeros", mason: "the one whose operands are least (min) or greatest (max)", wayOut: "round(min(a / b), n)" },
     "numeric-stddev": { kind: "differs", what: "stddev / variance of DISTINCT integers or numerics, or of a numeric declared without a precision or with more than 18 digits: the last digits", postgres: "exact numeric arithmetic, to its own scale", mason: "a double's 15 digits", wayOut: "round(stddev(x), 10)" },
+    "jsonb-equal-text": { kind: "differs", what: "GROUP BY or DISTINCT of jsonb values that are equal but written apart ('1.0' and '1.00', '{\"a\": 1}' and '{\"a\": 1.0}')", postgres: "one group, written as the value its plan reads first", mason: "one group, written as the value DuckDB reads first: either may be any of them", wayOut: "GROUP BY j::text where the text must decide", heldBy: "tests/duck-jsonb-order.test.ts" },
     "char-n-minmax": { kind: "differs", what: "greatest / least of a char(n) beside a text", postgres: "compares without the padding, answers the first of equal values", mason: "compares the padded text", wayOut: "cast the char(n) to text first" },
     "nested-exact-numeric": { kind: "differs", what: "an expression over a numeric a subquery computed exactly (a division, an avg, a round), outside that subquery: SELECT x * 2 FROM (SELECT a / 3 AS x …)", postgres: "exact digits at every step", mason: "the subquery's value as DuckDB computed it (a double's 15 digits) where the outer query computes with it; passed through as it is (SELECT *, s.x), it is exact", wayOut: "compute it in the outermost query, or round(…, n) inside" },
     // ---------------------------------------------------------------------------------------------- limits

package/dist/src/duck/jsonb-order.js

@@ -0,0 +1,175 @@
+/**
+ * JSONB'S ORDER AND EQUALITY AS BYTES (a BI tool sorts any column a user clicks: `ORDER BY "e"."tags" DESC` over a jsonb
+ * column was refused until this). Postgres orders and equates jsonb by `compareJsonbContainers` (jsonb_util.c), which
+ * walks both values token by token and stops at the first difference:
+ *  - by TYPE first, at any depth: null < string < number < boolean < array < object (jbvType's order);
+ *  - an array with more elements sorts higher, an object with more pairs higher; of equal counts, element by element,
+ *    pair by pair in STORAGE order (a key shorter in bytes first, then by its bytes; a key held twice keeps its LAST
+ *    value), each pair's key then its value;
+ *  - a number as a numeric (`10 > 9.5`, `1.0 = 1.00`, `1e2 = 100`, `-0 = 0`), a boolean false < true, a string — a
+ *    key's too — under the DATABASE's default collation (`varstr_cmp(..., DEFAULT_COLLATION_OID)`), whatever a
+ *    column's own collation, ties broken by the bytes;
+ *  - THE RAW SCALAR: a scalar at the top is stored as a one-element array marked `rawScalar`, and the two are compared
+ *    as arrays — the marks first (a raw scalar below an array), then, overriding that, the counts — so `[] < 1 < [1]`
+ *    (the source's comment: "a mild anomaly that an empty top level array sorts less than null"), and every top-level
+ *    scalar sorts below every object.
+ * Equality is that comparison's 0 (`jsonb_eq`), so it is the same key, never the canonical text: `1.0` and `1.00` are
+ * one jsonb written two ways.
+ *
+ * The key is one byte string whose plain byte order (DuckDB's BLOB comparison: bytes, then the shorter first) is that
+ * order, and whose equality is that equality. Every value is prefix-free (it ends where its own encoding says), so a
+ * container is its tag, its count and its members one after another, and the first differing byte is the first token
+ * Postgres finds different:
+ *  - a tag per type, in jbvType's order: null 0x10, string 0x20, number 0x30, boolean 0x40, array 0x50, object 0x60;
+ *  - an array: its tag, its count (4 bytes, big-endian), then 0x00 for a raw scalar and 0x01 for an array (the count
+ *    first: it decides before the mark), then each element; an object: its tag, its count, each pair's key and value;
+ *  - a number: 0x80 for zero; else a sign byte (0x81 positive, 0x7f negative), the decimal exponent E of 0.d₁d₂… × 10^E
+ *    (4 bytes, biased), the digits without leading or trailing zeros, a 0x00 — every byte after the sign inverted for a
+ *    negative number, so a larger magnitude sorts lower;
+ *  - a string: the collation's key of it, then a 0x00 (a jsonb string holds no NUL): its UTF-8 bytes under C (`bytes`);
+ *    under ICU, ICU's sort key (DuckDB's `icu_sort_key`, which the rewrite hands in) and the bytes' hex; under a libc
+ *    locale the measured key (src/duck/libc-order.ts `libcKey`) and the bytes' hex, as the rewrite sorts text
+ *    (src/duck/rewrite.ts `libcKey`).
+ * A string's collation key is computed by DuckDB's ICU for the ICU and libc kinds, which a JavaScript function cannot
+ * call: the rewrite passes the document's strings' ICU keys as a list (`jsonbStrings` lists the strings, in the order
+ * this walk meets them; `icuKeys` the list DuckDB made of them). Held to Postgres by tests/duck-jsonb-order.test.ts
+ * (PGlite's order) and the oracle (tests/integration/duck-oracle.itest.ts, every collation it is run under).
+ */
+import { Buffer } from "node:buffer";
+import { parseJson, storedPairs } from "./jsonb.js";
+import { libcKept, libcKey } from "./libc-order.js";
+const TAG = { null: 0x10, str: 0x20, num: 0x30, bool: 0x40, arr: 0x50, obj: 0x60 };
+const BIAS = 0x80000000;
+/** every string of the value — each object's keys and every string value — in the order the key's walk meets them */
+function eachString(v, out) {
+    switch (v.t) {
+        case "str":
+            out.push(v.v);
+            return;
+        case "arr":
+            for (const x of v.v)
+                eachString(x, out);
+            return;
+        case "obj":
+            for (const [k, x] of storedPairs(v.v)) {
+                out.push(k);
+                eachString(x, out);
+            }
+            return;
+        default: return;
+    }
+}
+/**
+ * THE FUNCTIONS A KEY IS MADE BY, per collation kind (src/duck/functions.ts registers them): the key's, and the one
+ * listing the strings DuckDB makes ICU keys of (none under C). A statement's key (src/duck/rewrite.ts `jsonbKey`) and a
+ * stored one (src/duck/kept.ts `orderKeySql`) both read this table, so the two are one computation by construction;
+ * a stored key calls each name with `STORED_SUFFIX`, the same function answering NULL where this one raises.
+ */
+export const JSONB_KEY_FUNCTIONS = {
+    bytes: { key: "__mason_jsonb_key", strings: null },
+    icu: { key: "__mason_jsonb_key_icu", strings: "__mason_jsonb_strings" },
+    libc: { key: "__mason_jsonb_key_libc", strings: "__mason_jsonb_libc_kept" },
+};
+export const STORED_SUFFIX = "_stored";
+/**
+ * THE KEY'S VERSION: a stored key (src/duck/kept.ts `orderId`) is this encoding's, so a release that changes a byte of
+ * it — a tag, the number's form, a string's — bumps this, and a key stored by an earlier one is dropped at the start
+ * and stored again (tests/duck-jsonb-order.test.ts holds the encoding; a change it catches is a change of this).
+ */
+export const KEY_VERSION = 1;
+/** The strings of a jsonb's text in the key's order (what DuckDB's ICU makes the keys of). */
+export function jsonbStrings(text) {
+    const out = [];
+    eachString(parseJson(text), out);
+    return out;
+}
+/** whether a jsonb's strings hold a character a libc order does not cover: what its key refuses by name
+ *  (`libc-uncovered`, src/duck/libc-order.ts `libcKey`), told from any other failure of the key (src/duck/kept.ts) */
+export function libcUncovered(order, text) {
+    return jsonbStrings(text).some((s) => { for (const ch of s)
+        if (order.uncovered[ch.codePointAt(0)])
+            return true; return false; });
+}
+/** The strings ICU's accent and case levels read for a libc key of each (src/duck/libc-order.ts `libcKept`). */
+export function jsonbLibcKept(order, text) {
+    return jsonbStrings(text).map((s) => libcKept(order, s));
+}
+const u32 = (n) => [(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff];
+/**
+ * A JSON number's text as its key: zero, or its sign, exponent and digits (the value's, exactly: no rounding through a
+ * double, and the scale left out — `1.0` and `1.00` are one number). Read from the text itself, never through a
+ * BigInt of its digits (`1e100000` is one digit and an exponent).
+ */
+export function numberKey(text) {
+    const m = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(text);
+    if (m === null)
+        throw new Error(`not a JSON number: ${JSON.stringify(text)}`);
+    const fraction = m[3] ?? "";
+    // the value is all × 10^(exponent - fraction's length), and so 0.<digits> × 10^E with E = all's length less that
+    const all = (m[2] + fraction).replace(/^0+/, "");
+    const digits = all.replace(/0+$/, "");
+    if (digits === "")
+        return [0x80];
+    const e = all.length - fraction.length + Number(m[4] ?? 0);
+    const body = [...u32(e + BIAS), ...[...digits].map((d) => d.charCodeAt(0)), 0x00];
+    return m[1] === "-" ? [0x7f, ...body.map((b) => 0xff - b)] : [0x81, ...body];
+}
+const hex = (s) => Buffer.from(s, "utf8").toString("hex");
+/** The key of a jsonb's text under the database's collation (the module's header says what it encodes). */
+export function jsonbKey(text, c) {
+    const out = [];
+    let at = 0;
+    const icuKey = () => {
+        if (c.kind === "bytes")
+            throw new Error("a C collation's key needs no ICU key");
+        const k = c.icuKeys[at++];
+        if (k === undefined)
+            throw new Error(`the jsonb holds more strings than the ${c.icuKeys.length} ICU keys handed in: a Mason bug (the rewrite's list and this walk disagree)`);
+        return k;
+    };
+    const str = (s) => {
+        const key = c.kind === "bytes" ? s : c.kind === "icu" ? `${icuKey()} ${hex(s)}` : `${libcKey(c.order, s, icuKey())} ${hex(s)}`;
+        for (const b of Buffer.from(key, "utf8"))
+            out.push(b);
+        out.push(0x00);
+    };
+    const value = (v) => {
+        switch (v.t) {
+            case "null":
+                out.push(TAG.null);
+                return;
+            case "bool":
+                out.push(TAG.bool, v.v ? 1 : 0);
+                return;
+            case "num":
+                out.push(TAG.num, ...numberKey(v.text));
+                return;
+            case "str":
+                out.push(TAG.str);
+                str(v.v);
+                return;
+            case "arr":
+                out.push(TAG.arr, ...u32(v.v.length), 0x01);
+                for (const x of v.v)
+                    value(x);
+                return;
+            case "obj": {
+                const pairs = storedPairs(v.v);
+                out.push(TAG.obj, ...u32(pairs.length));
+                for (const [k, x] of pairs) {
+                    str(k);
+                    value(x);
+                }
+                return;
+            }
+        }
+    };
+    const root = parseJson(text);
+    // a scalar at the top is a raw scalar: a one-element array marked as one (0x00, below an array's 0x01)
+    if (root.t !== "arr" && root.t !== "obj")
+        out.push(TAG.arr, ...u32(1), 0x00);
+    value(root);
+    if (c.kind !== "bytes" && at !== c.icuKeys.length)
+        throw new Error(`the jsonb holds ${at} strings, ${c.icuKeys.length} ICU keys were handed in: a Mason bug (the rewrite's list and this walk disagree)`);
+    return Buffer.from(out);
+}

package/dist/src/duck/jsonb.js

@@ -158,11 +158,19 @@ export function quoteJson(s) {
     }
     return `${out}"`;
 }
-/** jsonb's key order: shorter first, then by the UTF-8 bytes */
+/** jsonb's key order: shorter first, then by the UTF-8 bytes (lengthCompareJsonbStringValue) */
 function keyOrder(a, b) {
     const x = Buffer.from(a, "utf8"), y = Buffer.from(b, "utf8");
     return x.length - y.length || Buffer.compare(x, y);
 }
+/** an object's pairs as jsonb stores them, in its key order: a key held twice keeps its LAST value
+ *  (uniqueifyJsonbObject). What jsonb writes and what its order compares (src/duck/jsonb-order.ts) both read it. */
+export function storedPairs(members) {
+    const last = new Map();
+    for (const [k, x] of members)
+        last.set(k, x);
+    return [...last.keys()].sort(keyOrder).map((k) => [k, last.get(k)]);
+}
 /** a parsed value as jsonb writes it */
 export function writeJson(v) {
     switch (v.t) {
@@ -171,13 +179,7 @@ export function writeJson(v) {
         case "num": return numericText(parseNumeric(v.text));
         case "str": return quoteJson(v.v);
         case "arr": return `[${v.v.map(writeJson).join(", ")}]`;
-        case "obj": {
-            // a duplicate key keeps its LAST value (jsonb's uniqueifyJsonbObject)
-            const last = new Map();
-            for (const [k, x] of v.v)
-                last.set(k, x);
-            return `{${[...last.keys()].sort(keyOrder).map((k) => `${quoteJson(k)}: ${writeJson(last.get(k))}`).join(", ")}}`;
-        }
+        case "obj": return `{${storedPairs(v.v).map(([k, x]) => `${quoteJson(k)}: ${writeJson(x)}`).join(", ")}}`;
     }
 }
 /** A JSON text as Postgres's jsonb writes it; throws a json syntax error for a text that is no JSON. */

package/dist/src/duck/kept-route.js

@@ -32,7 +32,7 @@ export function canonicalRead(r) {
 }
 /** a read as one string: the request log's key (its `->>` or `->` included: the policy counts both for one path) */
 export const readKey = (r) => { const c = canonicalRead(r); return JSON.stringify([c.source, c.column, c.steps, c.either, c.canonical, c.asText]); };
-/** a read's key, back (null for a string that is not one) */
+/** a read's key, back (null for a string that is not one, and for a read of a kind this Mason does not know: `knownSteps`) */
 export function readOfKey(key) {
     let v;
     try {
@@ -46,8 +46,27 @@ export function readOfKey(key) {
     const [source, column, steps, either, canonical, asText] = v;
     if (typeof source !== "string" || typeof column !== "string" || typeof steps !== "string" || typeof either !== "boolean" || typeof canonical !== "boolean" || typeof asText !== "boolean")
         return null;
+    if (!knownSteps(steps))
+        return null;
     return canonicalRead({ source, column, steps, either, canonical, asText });
 }
+/**
+ * WHETHER A READ'S STEPS ARE A KIND THIS MASON KNOWS: a path's (a JSON array of keys and indexes) or a jsonb's key
+ * (`isKeyRead`). A request log or a ledger written by another release may hold a kind this one does not know (an
+ * earlier release met `equal:…` and `order:…` and failed every write of the table, and its status, parsing them as a
+ * path): such a read is INERT, never fatal — not counted (`readOfKey`), its ledger row dropped at the start with its
+ * column (src/duck/kept.ts `open`), and said raw (`pathText`).
+ */
+export function knownSteps(steps) {
+    if (isKeyRead({ steps }))
+        return true;
+    try {
+        return Array.isArray(JSON.parse(steps));
+    }
+    catch {
+        return false;
+    } // not JSON: not a path, so not a kind known here
+}
 /**
  * THE STORED PATH A READ IS ANSWERED FROM: its canonical read (`canonicalRead`) without `->>`. One column serves a
  * path's `->` and its `->>` (the test server: a popular key was stored twice, as `-> 'k'` and `->> 'k'`, two
@@ -72,9 +91,48 @@ export const hiddenName = (r) => `${HIDDEN_PREFIX}${createHash("sha256").update(
  * table name, so `public.orders.id` and `orders.id` still bind.
  */
 export const VISIBLE_CATALOG = "__mason_visible";
+/**
+ * A JSONB'S ORDER, AS A READ (src/duck/jsonb-order.ts; docs/DESIGN.md "jsonb's order"): `ORDER BY j`, `GROUP BY j`,
+ * `j = k` compare a key of the whole document, computed per row by a function — 19 s for `ORDER BY j LIMIT 100` over
+ * 20 M rows under C, 83 s under a libc locale, where the same sort from a stored key takes 0.1 s. So the key is a read
+ * like a path's, recorded, weighed and stored by the same keeper: its `steps` is `ORDER_STEPS` (never a path's, which
+ * is a JSON array). The rewrite records it as `ORDER_STEPS`; the keeper stores it as `ORDER_STEPS:<the collation's
+ * id>` (src/duck/kept.ts `own`), because a string's key is its collation's, so a key stored under another collation is
+ * another column, never read.
+ */
+export const ORDER_STEPS = "order";
+/** a stored key's marks (hex, DuckDB's `unhex`): a NULL document's, and a row whose key raised, computed when read. No
+ *  key begins with a 0x00 (src/duck/jsonb-order.ts: every key starts with an array's or an object's tag) */
+export const KEY_NULL_HEX = "00";
+export const KEY_RECOMPUTE_HEX = "0021";
+export const isOrderRead = (r) => r.steps === ORDER_STEPS || r.steps.startsWith(`${ORDER_STEPS}:`);
+/**
+ * A JSONB'S EQUALITY, AS A READ, under a LIBC collation alone (src/duck/rewrite.ts `equalKey`): `GROUP BY j`, `DISTINCT
+ * j`, `j = k`, `j IN (…)`, a join on `=`, `count(DISTINCT j)` compare the C key — a libc order's key refuses a string
+ * its measured order does not cover, where equality never needed an order (docs/DESIGN.md "jsonb's order"), so the
+ * order's stored key cannot serve them, and over 20 M rows a GROUP BY computing the C key per row took 32–45 s of CPU
+ * (34–77 s wall; docs/DESIGN.md "Under libc, the C key stored for equality"), 0.15–0.23 s from it. So the C key
+ * is a read of its own, stored beside the order's as `EQUAL_STEPS:<id>` (src/duck/kept.ts `equalId`). Under C and ICU
+ * the order's key equates exactly and never refuses, so equality reads IT and this read is never recorded nor stored
+ * (src/duck/kept.ts `own` folds one into the order's).
+ */
+export const EQUAL_STEPS = "equal";
+export const isEqualRead = (r) => r.steps === EQUAL_STEPS || r.steps.startsWith(`${EQUAL_STEPS}:`);
+/** a jsonb's key, the order's or the equality's: a BLOB column of the key's bytes, its marks `KEY_NULL_HEX` and
+ *  `KEY_RECOMPUTE_HEX`, stored under an id of what its bytes depend on */
+export const isKeyRead = (r) => isOrderRead(r) || isEqualRead(r);
 /** a read as a developer writes it: `payload ->> 'customer_id'`, `payload #> '{a,0}'`; a stored path (no `asText`) as
- *  its `->` read */
+ *  its `->` read; a jsonb's order and equality as what reads each (under libc the order's key sorts and the C key
+ *  equates, two columns: `mason status` names each by its own) */
 export function pathText(r) {
+    // a kind this Mason does not know (another release's): said as it is stored, never parsed (`knownSteps`)
+    if (!knownSteps(r.steps))
+        return `${r.column} (stored as ${r.steps}: a kind this Mason does not know)`;
+    if (isEqualRead(r))
+        return `GROUP BY ${r.column} (its jsonb equality: grouped, DISTINCT, = and IN)`;
+    if (isOrderRead(r))
+        return r.steps.startsWith(`${ORDER_STEPS}:libc:`) ? `ORDER BY ${r.column} (its jsonb order: sorted, and compared by < and >)`
+            : `ORDER BY ${r.column} (its jsonb order: sorted, grouped and compared)`;
     const steps = JSON.parse(r.steps);
     const op = (one) => one ? (r.asText ? "->>" : "->") : (r.asText ? "#>>" : "#>");
     if (steps.length === 1 && !r.either) {
@@ -84,6 +142,28 @@ export function pathText(r) {
     const element = (s) => typeof s === "number" || /^[A-Za-z0-9_]+$/.test(s) ? String(s) : `"${String(s).replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"`;
     return `${r.column} ${op(false)} '{${steps.map(element).join(",").replaceAll("'", "''")}}'`;
 }
+/**
+ * WHAT AN EXACT READ RAISES AT A MARK (src/duck/rewrite.ts `keptRead`, `keptKey`): a row marked to be computed
+ * (`KEPT_RECOMPUTE`, `KEY_RECOMPUTE_HEX`) written AFTER the path was counted exact — by the stream, the poll or a copy's
+ * side table, before the detector (src/duck/kept.ts `check`) counts it and turns the reads to the CASE — is never read
+ * as a value: the exact read raises at it, by the column alone. A jsonb's key under a libc collation is marked where
+ * its strings hold a character the measured order does not cover, the gap the statement's own key refuses by name
+ * (`libc-uncovered`); any other mark is a Mason bug. Once counted (a minute at most), the CASE computes the row from its
+ * document, and raises the function's own error (the character named) or answers. Raised by `MARKED_FUNCTIONS`, never
+ * DuckDB's `error()`, which is volatile (a window's ORDER BY refuses it: the eager oracle's `rank() OVER (ORDER BY j)`).
+ */
+export function markedError(r, libc) {
+    if (isOrderRead(r) && libc) {
+        return { code: "0A000", message: "a jsonb compared or sorted here holds a string Mason cannot order as the database's collation does (a combining mark, a Thai or " +
+                "Lao vowel written before its consonant, a private-use or unassigned code point), in a row written since its stored keys were counted; asked again " +
+                "in a minute, the error names the character (Mason gap libc-uncovered)" };
+    }
+    return { code: "XX000", message: `a stored value of ${pathText(r)} could not be computed when its row was written (a Mason bug: report it, with mason logs); ` +
+            "asked again in a minute, it is computed from the row's document" };
+}
+/** the functions that raise `markedError` at a marked row (src/duck/functions.ts): of the column read (a key as its hex,
+ *  so neither is a constant DuckDB would fold), the code and the message; a key's answers a BLOB, a path's a VARCHAR */
+export const MARKED_FUNCTIONS = { key: "__mason_marked_key", value: "__mason_marked_value" };
 const isObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
 const kindOf = (n) => isObject(n) && Object.keys(n).length === 1 && /^[A-Z]/.test(Object.keys(n)[0]) ? Object.keys(n)[0] : null;
 const bodyOf = (n) => n[kindOf(n)];