@mason-data/cli 0.8.0 → 0.10.0

package/dist/src/duck/grants.js

@@ -22,7 +22,7 @@
  * table the role has only part of is refused, never narrowed (a narrowed `*` would change the result's columns); a
  * system column (xmin, ctid) needs the whole table.
  */
-import { StatementError } from "../api/postgres/server.js";
+import { StatementError } from "./statement.js";
 import { catalogRows } from "./analyze.js";
 /** no roles file: every session reads every copied table, as before; one object, so the listener sees no change */
 export const NO_ROLES = { file: null, grants: new Map(), problem: null, default: null };

package/dist/src/duck/held-types.js

@@ -0,0 +1,64 @@
+/**
+ * WHAT THE COPY HOLDS FOR A POSTGRES TYPE (issues 126 and 129): the one list that says so, read by the copy
+ * (src/duck/sync.ts: what it reads and declares), the catalog (the column declared as the source spells it, always), the
+ * wire (src/duck/types.ts: the source's OID) and the rewrite (src/duck/rewrite.ts: a literal, a sort).
+ *
+ * THE CLASS: a type the copy holds as another type. The catalog said the wire's type, and the wire said what DuckDB
+ * happened to read (`timetz` as text over DuckDB's TIMETZ, `point` as text over a STRUCT, `box` not copied at all), so a
+ * tool believing the catalog sent text functions, and they failed in the engine's words.
+ *
+ * THE DECISION: the catalog reports the source's type, whatever the copy holds, so Postgres's own analysis decides every
+ * statement; and every type is one of two kinds:
+ *  - `same` (`NATIVE`): DuckDB holds a value of the type itself, whose order and text are Postgres's;
+ *  - `text`: DuckDB holds Postgres's text of the value (`c::text`, an array as `c::text[]`). That is anything `NATIVE`
+ *    does not name, a type added to Postgres or made by a user included: nothing falls through to what DuckDB reads.
+ * What can be answered over a text is answered (the value, `::text`, `=` and GROUP BY where equal values have one text); an
+ * order is not the text's, so a sort is refused by name (`held-type-compare`), and every operator, aggregate and cast
+ * stays refused until the oracle has verified it (src/duck/verified.ts).
+ *
+ * `tests/duck-held-types.test.ts` holds this list to pg_type: a type of Postgres's named neither way fails it.
+ */
+/** the types DuckDB holds as themselves, as `format_type` spells them with their typmod left off */
+const NATIVE = new Set(["smallint", "integer", "bigint", "real", "double precision", "numeric", "text", "character varying", "character", "name", '"char"',
+    "boolean", "uuid", "date", "time without time zone", "time with time zone", "timestamp without time zone", "timestamp with time zone", "interval",
+    "json", "jsonb", "bytea", "oid",
+    // held as their text from the first: an inet's output is the rewrite's and src/duck/types.ts's, a citext's its bytes
+    "inet", "citext"]);
+const held = (oid, array, size, textEqual = true, literal = true) => ({ oid, array, size, textEqual, literal });
+/** every built-in type a column can have that `NATIVE` does not name, by its pg_type.typname */
+export const HELD = {
+    point: held(600, 1017, 16, false), lseg: held(601, 1018, 32, false), path: held(602, 1019, -1, false), box: held(603, 1020, 32, false), polygon: held(604, 1027, -1, false),
+    line: held(628, 629, 24, false), circle: held(718, 719, 24, false), money: held(790, 791, 8, true, false), macaddr: held(829, 1040, 6), macaddr8: held(774, 775, 8), cidr: held(650, 651, -1),
+    bit: held(1560, 1561, -1), varbit: held(1562, 1563, -1), tsvector: held(3614, 3643, -1), tsquery: held(3615, 3645, -1, false), xml: held(142, 143, -1, false),
+    jsonpath: held(4072, 4073, -1, false), tid: held(27, 1010, 6), xid: held(28, 1011, 4), cid: held(29, 1012, 4), xid8: held(5069, 271, 8), pg_lsn: held(3220, 3221, 8),
+    regclass: held(2205, 2210, 4), regtype: held(2206, 2211, 4), regproc: held(24, 1008, 4), regprocedure: held(2202, 2207, 4), regoper: held(2203, 2208, 4), regoperator: held(2204, 2209, 4),
+    regconfig: held(3734, 3735, 4), regdictionary: held(3769, 3770, 4), regnamespace: held(4089, 4090, 4), regrole: held(4096, 4097, 4), regcollation: held(4191, 4192, 4),
+    int4range: held(3904, 3905, -1), int8range: held(3926, 3927, -1), daterange: held(3912, 3913, -1), tsrange: held(3908, 3909, -1), tstzrange: held(3910, 3911, -1, false),
+    numrange: held(3906, 3907, -1, false), int4multirange: held(4451, 6150, -1), int8multirange: held(4536, 6157, -1), datemultirange: held(4535, 6155, -1), tsmultirange: held(4533, 6152, -1),
+    tstzmultirange: held(4534, 6153, -1, false), nummultirange: held(4532, 6151, -1, false), int2vector: held(22, 1006, -1), oidvector: held(30, 1013, -1), aclitem: held(1033, 1034, 16),
+    refcursor: held(1790, 2201, -1), pg_snapshot: held(5038, 5039, -1, false), txid_snapshot: held(2970, 2949, -1, false),
+};
+/** `oid`'s array: DuckDB reads no array of it, so the copy holds it as a list of its digits */
+export const OID_ARRAY = 1028;
+const HELD_OIDS = new Map(Object.values(HELD).map((h) => [h.oid, h]));
+const HELD_ARRAYS = new Map([...Object.values(HELD).map((h) => [h.array, h]), [OID_ARRAY, held(26, OID_ARRAY, -1)]]);
+/** a built-in type held as its text, by its OID or its array's */
+export const heldType = (oid) => oid === null ? undefined : HELD_OIDS.get(oid) ?? HELD_ARRAYS.get(oid);
+/** a held type's name, for a refusal */
+export const heldName = (oid) => Object.entries(HELD).find(([, h]) => h.oid === oid || h.array === oid)?.[0] ?? (oid === 18 ? '"char"' : oid === OID_ARRAY ? "oid[]" : String(oid));
+/** an array of a type held as its text: a list of each element's text */
+export const isHeldArray = (oid) => oid !== null && HELD_ARRAYS.has(oid);
+/** a time written in the session's zone, which the copy's text cannot be: written again where the statement executes */
+export const ZONED_HELD = new Map([[3910, "tstzrange"], [4534, "tstzmultirange"]]);
+/** what `format_type` writes, its typmod and `[]` left off (`time(3) with time zone`, `bit(8)`, `numeric(12,2)[]`) */
+const baseOf = (spelled) => spelled.replace(/\[\]$/, "").replace(/\(\d+(,\d+)?\)/, "").replace(/\s+/g, " ").trim().replace(/^interval .*/, "interval");
+/** whether DuckDB holds a value of this type as itself */
+export const isNative = (spelled) => NATIVE.has(baseOf(spelled));
+/** How the copy holds a column of the source's type `spelled` (`typeKind`: pg_type.typtype of the type or its element).
+ *  An enum is DuckDB's ENUM; any other type a database made is its text. */
+export function holdOf(spelled, typeKind) {
+    const array = spelled.endsWith("[]");
+    if (typeKind === "e" || (isNative(spelled) && !(array && baseOf(spelled) === "oid")))
+        return "same";
+    return array ? "text[]" : "text";
+}

package/dist/src/duck/history.js

@@ -2,6 +2,10 @@ import { MASON_SCHEMA } from "./store.js";
 /** How many days every history table keeps (the request log, the sync's events, the usage samples, the copies'
  *  sizes) unless MASON_HISTORY_DAYS says. */
 export const DEFAULT_HISTORY_DAYS = 7;
+/** The instant `days` days before `now` (ms since the epoch), ISO 8601 in UTC: what a history table's trim deletes
+ *  before. `now` is the module's own clock, the one that stamps its rows, never the engine's `now()`: a module reads
+ *  ONE clock, or a clock it is given (a test's, a run's) and the engine's disagree and the trim deletes what it keeps. */
+export function cutoff(now, days) { return new Date(now - days * 86_400_000).toISOString(); }
 const KEPT = "the request log, the sync's events, the usage samples and the synced tables' sizes";
 /** The days MASON_HISTORY_DAYS keeps, or what is wrong with the text. Unset or empty: the default. */
 export function historyDays(text) {

package/dist/src/duck/jsonb-order.js

@@ -30,14 +30,14 @@
  *    under ICU, ICU's sort key (DuckDB's `icu_sort_key`, which the rewrite hands in) and the bytes' hex; under a libc
  *    locale the measured key (src/duck/libc-order.ts `libcKey`) and the bytes' hex, as the rewrite sorts text
  *    (src/duck/rewrite.ts `libcKey`).
- * A string's collation key is computed by DuckDB's ICU for the ICU and libc kinds, which a JavaScript function cannot
+ * A string's collation key is computed by DuckDB's ICU for the ICU kind, which a JavaScript function cannot
  * call: the rewrite passes the document's strings' ICU keys as a list (`jsonbStrings` lists the strings, in the order
  * this walk meets them; `icuKeys` the list DuckDB made of them). Held to Postgres by tests/duck-jsonb-order.test.ts
  * (PGlite's order) and the oracle (tests/integration/duck-oracle.itest.ts, every collation it is run under).
  */
 import { Buffer } from "node:buffer";
 import { parseJson, storedPairs } from "./jsonb.js";
-import { libcKept, libcKey } from "./libc-order.js";
+import { libcKey } from "./libc-order.js";
 const TAG = { null: 0x10, str: 0x20, num: 0x30, bool: 0x40, arr: 0x50, obj: 0x60 };
 const BIAS = 0x80000000;
 /** every string of the value — each object's keys and every string value — in the order the key's walk meets them */
@@ -68,7 +68,7 @@ function eachString(v, out) {
 export const JSONB_KEY_FUNCTIONS = {
     bytes: { key: "__mason_jsonb_key", strings: null },
     icu: { key: "__mason_jsonb_key_icu", strings: "__mason_jsonb_strings" },
-    libc: { key: "__mason_jsonb_key_libc", strings: "__mason_jsonb_libc_kept" },
+    libc: { key: "__mason_jsonb_key_libc" },
 };
 export const STORED_SUFFIX = "_stored";
 /**
@@ -86,14 +86,11 @@ export function jsonbStrings(text) {
 /** whether a jsonb's strings hold a character a libc order does not cover: what its key refuses by name
  *  (`libc-uncovered`, src/duck/libc-order.ts `libcKey`), told from any other failure of the key (src/duck/kept.ts) */
 export function libcUncovered(order, text) {
-    return jsonbStrings(text).some((s) => { for (const ch of s)
+    return jsonbStrings(text).some((s) => { if (order.unread)
+        return true; for (const ch of s)
         if (order.uncovered[ch.codePointAt(0)])
             return true; return false; });
 }
-/** The strings ICU's accent and case levels read for a libc key of each (src/duck/libc-order.ts `libcKept`). */
-export function jsonbLibcKept(order, text) {
-    return jsonbStrings(text).map((s) => libcKept(order, s));
-}
 const u32 = (n) => [(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff];
 /**
  * A JSON number's text as its key: zero, or its sign, exponent and digits (the value's, exactly: no rounding through a
@@ -120,15 +117,15 @@ export function jsonbKey(text, c) {
     const out = [];
     let at = 0;
     const icuKey = () => {
-        if (c.kind === "bytes")
-            throw new Error("a C collation's key needs no ICU key");
+        if (c.kind !== "icu")
+            throw new Error("only ICU's key is handed in");
         const k = c.icuKeys[at++];
         if (k === undefined)
             throw new Error(`the jsonb holds more strings than the ${c.icuKeys.length} ICU keys handed in: a Mason bug (the rewrite's list and this walk disagree)`);
         return k;
     };
     const str = (s) => {
-        const key = c.kind === "bytes" ? s : c.kind === "icu" ? `${icuKey()} ${hex(s)}` : `${libcKey(c.order, s, icuKey())} ${hex(s)}`;
+        const key = c.kind === "bytes" ? s : c.kind === "icu" ? `${icuKey()} ${hex(s)}` : `${libcKey(c.order, s)} ${hex(s)}`;
         for (const b of Buffer.from(key, "utf8"))
             out.push(b);
         out.push(0x00);
@@ -169,7 +166,7 @@ export function jsonbKey(text, c) {
     if (root.t !== "arr" && root.t !== "obj")
         out.push(TAG.arr, ...u32(1), 0x00);
     value(root);
-    if (c.kind !== "bytes" && at !== c.icuKeys.length)
+    if (c.kind === "icu" && at !== c.icuKeys.length)
         throw new Error(`the jsonb holds ${at} strings, ${c.icuKeys.length} ICU keys were handed in: a Mason bug (the rewrite's list and this walk disagree)`);
     return Buffer.from(out);
 }

package/dist/src/duck/kept-route.js

@@ -149,14 +149,14 @@ export function pathText(r) {
  * as a value: the exact read raises at it, by the column alone. A jsonb's key under a libc collation is marked where
  * its strings hold a character the measured order does not cover, the gap the statement's own key refuses by name
  * (`libc-uncovered`); any other mark is a Mason bug. Once counted (a minute at most), the CASE computes the row from its
- * document, and raises the function's own error (the character named) or answers. Raised by `MARKED_FUNCTIONS`, never
+ * document, and raises the function's own error or answers. Raised by `MARKED_FUNCTIONS`, never
  * DuckDB's `error()`, which is volatile (a window's ORDER BY refuses it: the eager oracle's `rank() OVER (ORDER BY j)`).
  */
 export function markedError(r, libc) {
     if (isOrderRead(r) && libc) {
-        return { code: "0A000", message: "a jsonb compared or sorted here holds a string Mason cannot order as the database's collation does (a combining mark, a Thai or " +
-                "Lao vowel written before its consonant, a private-use or unassigned code point), in a row written since its stored keys were counted; asked again " +
-                "in a minute, the error names the character (Mason gap libc-uncovered)" };
+        return { code: "0A000", message: "a jsonb compared or sorted here holds a string Mason cannot order as the database's collation does, " +
+                "in a row written since its stored keys were counted; asked again " +
+                "in a minute, the statement's own key answers or refuses it (Mason gap libc-uncovered)" };
     }
     return { code: "XX000", message: `a stored value of ${pathText(r)} could not be computed when its row was written (a Mason bug: report it, with mason logs); ` +
             "asked again in a minute, it is computed from the row's document" };
@@ -224,7 +224,8 @@ export function exposes(tree) {
             return false;
         const k = kindOf(v);
         const b = k === null ? v : bodyOf(v);
-        if (k === "SelectStmt" && Array.isArray(b.targetList) && directTables(b.fromClause).length > 0
+        // (an arm of a set operation is a SelectStmt with no wrapper of its own: k is null)
+        if ((k === "SelectStmt" || k === null) && Array.isArray(b.targetList) && directTables(b.fromClause).length > 0
             && b.targetList.some((t) => kindOf(t) === "ResTarget" && star(bodyOf(t).val)))
             return true;
         if (k === "JoinExpr" && b.isNatural === true && directTables([b.larg, b.rarg]).length > 0)

package/dist/src/duck/kept.js

@@ -2,8 +2,8 @@ import { DUCKDB_VERSION, MASON_SCHEMA } from "./store.js";
 import { createHash } from "node:crypto";
 import { canonicalRead, EQUAL_STEPS, HIDDEN_PREFIX, HIDDEN_RE, hiddenName, isEqualRead, isKeyRead, isOrderRead, KEY_NULL_HEX, KEY_RECOMPUTE_HEX, knownSteps, ORDER_STEPS, pathKey, pathText, readOfKey, VISIBLE_CATALOG } from "./kept-route.js";
 import { JSONB_KEY_FUNCTIONS, KEY_VERSION, libcUncovered, STORED_SUFFIX } from "./jsonb-order.js";
+import { TURN_FUNCTION } from "./functions.js";
 import { encodeLibcOrder } from "./libc-order.js";
-import { icuNameOf } from "./rewrite.js";
 /** a path read by this many statements in the request log is stored (a larger table's also needs `KEEP_FILL_SHARE`) */
 export const KEEP_AFTER_STATEMENTS = 3;
 /** a path of a SMALL table (`SMALL_FILL_SECONDS`) read by one statement that took this long (ms, the request log's
@@ -97,7 +97,7 @@ const tableOf = (source) => { const at = source.indexOf("."); return `${ident(so
 /** the value a stored column takes from its JSON column's `value` (a column reference or an expression): the path's
  *  `->` answer, never NULL (a NULL document's is `KEPT_NULL` too: the column's NULL is a row not stored yet) */
 export function keptValueSql(p, value) {
-    return `coalesce(__mason_json_kept(NULL, ${value}, ${literal(p.steps)}, ${p.either}, ${p.canonical}, false), ${NULL_SQL})`;
+    return `coalesce(__mason_json_kept(${TURN_FUNCTION}(NULL, ${value}), ${value}, ${literal(p.steps)}, ${p.either}, ${p.canonical}, false), ${NULL_SQL})`;
 }
 /**
  * A JSONB'S STORED KEY, from its column's `value` (src/duck/kept-route.ts `ORDER_STEPS`): the key the statement would
@@ -106,9 +106,9 @@ export function keptValueSql(p, value) {
  * `KEY_RECOMPUTE_HEX` where the key raised, which its read computes (and raises) again.
  */
 export function orderKeySql(c, value) {
-    const call = (name, ...args) => `${name}${STORED_SUFFIX}(NULL, ${args.join(", ")})`;
-    const key = c.kind === "bytes" ? call(JSONB_KEY_FUNCTIONS.bytes.key, value)
-        : call(JSONB_KEY_FUNCTIONS[c.kind].key, value, `list_transform(${call(JSONB_KEY_FUNCTIONS[c.kind].strings, value)}, __mason_s -> icu_sort_key(__mason_s, ${literal(icuNameOf(c))}))`);
+    const call = (name, ...args) => `${name}${STORED_SUFFIX}(${TURN_FUNCTION}(NULL, ${value}), ${args.join(", ")})`;
+    const key = c.kind !== "icu" ? call(JSONB_KEY_FUNCTIONS[c.kind].key, value)
+        : call(JSONB_KEY_FUNCTIONS.icu.key, value, `list_transform(${call(JSONB_KEY_FUNCTIONS.icu.strings, value)}, __mason_s -> icu_sort_key(__mason_s, ${literal(c.name)}))`);
     return `CASE WHEN (${value}) IS NULL THEN unhex('${KEY_NULL_HEX}') ELSE coalesce(${key}, unhex('${KEY_RECOMPUTE_HEX}')) END`;
 }
 /**
@@ -124,7 +124,7 @@ export function orderId(c, libc, versions = `k${KEY_VERSION}:${DUCKDB_VERSION}`)
         return `icu:${c.name}:${versions}`;
     if (libc === undefined)
         throw new Error("a libc collation's order was not read (Mason bug: report it)");
-    return `libc:${c.icu}:${createHash("sha256").update(encodeLibcOrder(libc)).digest("hex").slice(0, 16)}:${versions}`;
+    return `libc:${createHash("sha256").update(encodeLibcOrder(libc)).digest("hex").slice(0, 16)}:${versions}`;
 }
 /**
  * THE C KEY'S ID, stored under a libc collation for equality (`EQUAL_STEPS:<id>`, src/duck/kept-route.ts): the C
@@ -879,7 +879,9 @@ export class KeptJsonPaths {
         const of = read < marked ? ` (of the first ${read.toLocaleString("en-US")} read, of ${marked.toLocaleString("en-US")})` : "";
         const said = [];
         if (uncovered > 0)
-            said.push(`${uncovered.toLocaleString("en-US")} row(s) hold a character the database's libc order is not measured for, so a sort of them is refused by name (Mason gap libc-uncovered)`);
+            said.push(libc?.unread === true
+                ? `the database's collation is one Mason could not read, so a sort of ${uncovered.toLocaleString("en-US")} row(s) is refused by name (Mason gap libc-locale-unread)`
+                : `${uncovered.toLocaleString("en-US")} row(s) hold a character the database's libc order is not measured for, so a sort of them is refused by name (Mason gap libc-uncovered)`);
         if (read > uncovered)
             said.push(`${(read - uncovered).toLocaleString("en-US")} row(s) its function could not read are computed from their document when read (a Mason bug: report it, with mason logs)`);
         return said.join("; ") + of;