@mason-data/cli 0.8.0 → 0.10.0

package/dist/src/duck/plan.js

@@ -6,16 +6,20 @@
  * per text until the schema changes.
  */
 import { deparseSync, QuoteUtils } from "pgsql-deparser";
-import { gapRefusal, StatementError } from "../api/postgres/server.js";
+import { HELD_ROW_INPUT, HELD_TYPE_INPUT } from "../api/postgres/schema.js";
+import { heldName, heldType, isHeldArray } from "./held-types.js";
+import { collationOids, functionsCalled, outputExpressions, textOrdering } from "./pgtree.js";
+import { gapRefusal, StatementError } from "./statement.js";
 import { parseSql } from "../api/postgres/parser.js";
 import { analyze, catalogRows, readParameters } from "./analyze.js";
-import { describeKeys, nameUserKeys, resolvedKeys } from "./resolved.js";
+import { argumentTypes, describeKeys, nameUserKeys, heldUses, nullTested, resolvedKeys } from "./resolved.js";
 import { expandViews, MAX_VIEW_DEPTH, viewRefused } from "./views.js";
 import { selectListAfterLimit } from "./late.js";
+import { libcBounded } from "./libc-bound.js";
 import { pgTyplen } from "./types.js";
-import { datumText, masonCalls, rewrite, treeDifference, Unrewritable } from "./rewrite.js";
+import { giveTurns, datumText, masonCalls, rewrite, treeDifference, Unrewritable } from "./rewrite.js";
 import { exposes, hideKept, readKey, searchSchemas } from "./kept-route.js";
-import { distinctCountTables, distinctKeyCounts, uniqueColumnsOf, uniqueColumnsSql } from "./distinct-key.js";
+import { distinctCountTables, distinctKeyCounts, fanOutTables, groupedLimitTables, primaryKeysOf, uniqueColumnsOf, uniqueColumnsSql } from "./distinct-key.js";
 import { enforceGrants } from "./grants.js";
 import { attributeRefused, expandRowFilters, wallStatistics, wallSwitch } from "./row-filter.js";
 const JSON_OID = 114, JSONB_OID = 3802;
@@ -44,6 +48,20 @@ async function distinctKeys(catalog, tree) {
         return new Set();
     return distinctKeyCounts(tree, uniqueColumnsOf(await catalogRows(catalog, uniqueColumnsSql(relids))));
 }
+/** the primary keys that may prove a LIMIT with no ORDER BY's joins bounded, or a grouped LIMIT grouped by a whole key
+ *  (src/duck/late.ts), read only for a statement that has one */
+async function joinKeys(catalog, tree) {
+    const relids = [...new Set([...fanOutTables(tree), ...groupedLimitTables(tree)])].sort((a, b) => a - b);
+    return relids.length === 0 ? new Map() : primaryKeysOf(await catalogRows(catalog, uniqueColumnsSql(relids)));
+}
+/** which of `functions` the catalog says are volatile: asked only of a statement with a derived read the list after the
+ *  limit may read twice (src/duck/late.ts) */
+async function volatileFunctions(catalog, functions) {
+    if (functions.length === 0)
+        return new Set();
+    const rows = await catalogRows(catalog, `SELECT oid::text FROM pg_proc WHERE provolatile = 'v' AND oid IN (${functions.map((f) => String(Math.trunc(f))).join(", ")})`);
+    return new Set(rows.map(([oid]) => Number(oid)));
+}
 /**
  * DuckDB's keywords that Postgres's are not: a column named `at` (DuckDB's AT TIME ZONE) or `qualify` is an identifier
  * the deparser writes bare, as Postgres reads it, and DuckDB refuses as a syntax error. The deparser's one quoting
@@ -60,9 +78,17 @@ export function quoteDuckKeywords(words) {
         return q === ident && duckKeywords.has(ident) ? `"${ident}"` : q;
     };
 }
-/** a search_path as SET writes it: each schema quoted (`"$user"` kept) */
-const pathSql = (searchPath) => searchPath.split(",").map((s) => s.trim()).filter((s) => s !== "")
-    .map((s) => s.startsWith('"') ? s : `"${s.replaceAll('"', '""')}"`).join(", ") || "public";
+/**
+ * EVERY TYPE THAT HOLDS A CITEXT, to any depth, by the catalog's structure: citext itself; an array by its element
+ * (`typelem`, a nested array is the same type); a domain by its base (`typbasetype`); a composite by its attributes; a
+ * range by its subtype and a multirange by its range (PR 674 shrink 2: an array of citext compared by bytes in a
+ * quantified subquery). One recursive term, since Postgres takes the recursive reference once.
+ */
+export const CITEXT_HELD = "WITH RECURSIVE held(oid) AS (SELECT oid FROM pg_type WHERE typname = 'citext' UNION " +
+    "SELECT t.oid FROM pg_type t, held h WHERE t.typelem = h.oid OR t.typbasetype = h.oid " +
+    "OR (t.typrelid <> 0 AND EXISTS (SELECT 1 FROM pg_attribute a WHERE a.attrelid = t.typrelid AND a.attnum > 0 AND NOT a.attisdropped AND a.atttypid = h.oid)) " +
+    "OR EXISTS (SELECT 1 FROM pg_range r WHERE (r.rngtypid = t.oid AND r.rngsubtype = h.oid) OR (r.rngmultitypid = t.oid AND r.rngtypid = h.oid))) " +
+    "SELECT oid::text FROM held";
 /**
  * `verified`: what the oracle has held to Postgres (src/duck/verified.ts). A statement using anything else is REFUSED,
  * by name — never answered by a rewrite nobody checked. null only for the oracle itself, which records what it verifies.
@@ -72,12 +98,18 @@ const pathSql = (searchPath) => searchPath.split(",").map((s) => s.trim()).filte
  * src/duck/analyze.ts), written back in a form DuckDB reads to the same value. A value Postgres refuses is refused with
  * its error (`'1.5'` for a smallint: 22P02).
  */
-export async function duckParameters(catalog, oids, values, timeZone) {
-    const read = await readParameters(catalog, oids, values, timeZone);
+export async function duckParameters(catalog, oids, values, under) {
+    const read = await readParameters(catalog, oids, values, under);
     if ("refused" in read) {
         const f = read.refused;
         throw new StatementError({ code: f.C ?? "22P02", message: f.M ?? "invalid parameter value", ...(f.D ? { detail: f.D } : {}), ...(f.H ? { hint: f.H } : {}) }, "");
     }
+    // a text[] value DuckDB's list cannot hold as Postgres read it: refused by name, as its literal is
+    if ("gap" in read) {
+        throw read.gap === "multidim-array"
+            ? gapRefusal("multidim-array", `a multi-dimensional array as parameter $${read.parameter} is not supported by Mason`, "bind a one-dimensional array")
+            : gapRefusal("array-lower-bound", `an array whose subscripts do not start at 1 as parameter $${read.parameter} is not supported by Mason`, "bind the array without its bounds ('{a,b}', not '[2:3]={a,b}')");
+    }
     return values.map((v, i) => {
         const r = read.values[i];
         if (v === null || r === undefined || r === null)
@@ -167,31 +199,213 @@ function printed(tree, statement) {
     }
     throw new StatementError({ code: "0A000", message: "Mason cannot print this statement for the engine (a Mason gap: report it, with the statement)" }, statement);
 }
+/**
+ * A CHAIN'S CHILD READS ITS PARENT'S SIDE TABLE (src/duck/generated.ts, issue 124 stage 2): on the finished tree, each
+ * read of a table `relations` names is swapped for the table it maps to, in the same schema, under the name the statement
+ * read it by (its alias, else the table's own), so every column the rewrite qualified still binds. A bare name is the
+ * search path's first schema's (a generated table is planned under `public`), and a bare name a WITH in scope holds is a
+ * CTE's, never swapped. Every table named must be met: a read left on the served table would read the old parent, so
+ * one not met throws. The read-back below holds the swapped text to the tree, as any other.
+ */
+function swapRelations(tree, relations, schemas) {
+    const met = new Set();
+    const walk = (v, ctes) => {
+        if (Array.isArray(v)) {
+            for (const x of v)
+                walk(x, ctes);
+            return;
+        }
+        if (typeof v !== "object" || v === null)
+            return;
+        const o = v;
+        const r = o.RangeVar;
+        if (r !== undefined) {
+            if (r.schemaname === undefined && ctes.has(r.relname))
+                return;
+            const key = `${r.schemaname ?? schemas[0] ?? "public"}.${r.relname}`;
+            const to = relations.get(key);
+            if (to !== undefined) {
+                r.alias ??= { aliasname: r.relname };
+                r.relname = to;
+                met.add(key);
+            }
+            return;
+        }
+        // a column by the table's schema-qualified name (`public.counts.tenant_id`, valid Postgres): the swapped table is
+        // read under its own name as an alias, which binds no schema, so the column is named by that alias alone
+        const fields = o.ColumnRef?.fields;
+        if (fields !== undefined && fields.length >= 3) {
+            const sval = (x) => x?.String?.sval;
+            const schema = sval(fields[fields.length - 3]), table = sval(fields[fields.length - 2]);
+            if (schema !== undefined && table !== undefined && relations.has(`${schema}.${table}`))
+                o.ColumnRef.fields = fields.slice(-2);
+            return;
+        }
+        const list = o.withClause?.ctes;
+        const scope = list === undefined ? ctes : new Set([...ctes, ...list.map((c) => c.CommonTableExpr?.ctename ?? "")]);
+        for (const x of Object.values(o))
+            walk(x, scope);
+    };
+    walk(tree, new Set());
+    const missed = [...relations.keys()].filter((k) => !met.has(k));
+    if (missed.length > 0)
+        throw new Error(`a generated table's plan does not read ${missed.join(", ")} by a name it can swap (Mason bug: report it)`);
+}
+/** The catalog declares a type the source made with nothing over it (`PgColumn.made`), so what its analysis says is not
+ *  there (an `=`, an order, a field of a composite) the source may have: Mason's gap, never Postgres's refusal. */
+async function madeTypeRefusal(catalog, code, message, text) {
+    if (message === undefined || (code !== "42883" && code !== "42809" && code !== "42804" && code !== "42846"))
+        return null;
+    const types = (await catalogRows(catalog, `SELECT t.typname FROM pg_type t WHERE t.typinput::text LIKE '%${HELD_TYPE_INPUT}%'`)).map((r) => r[0]);
+    const name = types.find((t) => new RegExp(`\\b${t.replace(/[^\w]/g, ".")}\\b`).test(message));
+    if (name === undefined || /^function /.test(message) && !/\(.*\)/.test(message))
+        return null;
+    return gapRefusal("held-type-compare", `Mason cannot answer this over a column of the type ${name}: it holds the value's text, and has no operator of the type`, "read the value, or its text (x::text); compare, sort or take a field on the source's own columns", text);
+}
+/**
+ * A STATEMENT OF NO TABLE THAT ORDERS TEXT UNDER A COLLATION THAT IS NOT BYTE ORDER (issue 122): the catalog's database
+ * is C, so `SELECT 'a' < 'B'` answered there is byte order where the source's libc or ICU says otherwise. Its analysis
+ * (under the session's state) says where it orders a collatable value and under which collation (`textOrdering`); an
+ * operator or a function counts as ordering by the catalog's own name of it (`<`, `max` …, never `=` or `count`), and
+ * the resolver says whether that collation is byte order. "order": the statement is planned for the synced data, made
+ * and refused like any other. A statement the analysis refuses stays the catalog's, which raises Postgres's own error.
+ * "case" (issue 406): it calls upper, lower or initcap of text, under any collation (the catalog maps case by its own
+ * C.UTF-8's rules, never the source's ctype): the synced data's where its RUN answers the statement, which the caller
+ * decides (src/api/postgres/server.ts `answersCase`), since what the catalog answered before is kept where the synced
+ * data's plan or run fails.
+ */
+export async function ordersText(o) {
+    const analysis = await analyze(o.catalog, o.text, o.paramOids, o.under);
+    if ("refused" in analysis)
+        return null;
+    // the analysis as the session runs the statement: a simple Query binds no parameter, so one naming `$1` is Postgres's
+    // 42P02, which the catalog answers (the analysis above is a Parse's, which would infer its type)
+    if (!o.bindable && analysis.params.length > 0)
+        return null;
+    const oids = collationOids(analysis.tree);
+    if (oids.size === 0)
+        return null;
+    let resolved;
+    // (a collation the source lacks, or one Mason cannot follow: the synced data's plan refuses it by name, never the
+    // catalog's byte order; a nondeterministic one wherever it is named, equality and GROUP BY included)
+    try {
+        resolved = await o.collations.prepare((sql) => catalogRows(o.catalog, sql), oids);
+    }
+    catch (e) {
+        if (e instanceof Unrewritable)
+            return "order";
+        throw e;
+    }
+    // a statement of no table that holds a numeric (a literal such as 1.5 included) stays the catalog's, which computes it exactly, its text in byte
+    // order and its case as before (Mason gap fromless-numeric-text-order): the synced data holds a VALUES list's numeric
+    // otherwise than Postgres (issue 342's class), and the catalog answered it right
+    if ([...analysis.types.values()].some((ts) => ts.some((t) => t.type === NUMERIC || t.type === NUMERIC_ARRAY)))
+        return null;
+    const ordering = (coll) => {
+        try {
+            const c = resolved.resolve(coll);
+            return c !== null && c.kind !== "bytes";
+        }
+        catch (e) {
+            if (e instanceof Unrewritable)
+                return true;
+            throw e;
+        }
+    };
+    const { orders, operators, functions, calls } = textOrdering(analysis.tree);
+    if ([...orders].some(ordering))
+        return "order";
+    const ops = [...operators].filter(([, cs]) => [...cs].some(ordering)).map(([oid]) => oid);
+    const fns = [...functions].filter(([, cs]) => [...cs].some(ordering)).map(([oid]) => oid);
+    const cased = [...calls.keys()];
+    if (ops.length + fns.length + cased.length === 0)
+        return null;
+    const named = (await catalogRows(o.catalog, [
+        ...(ops.length > 0 ? [`SELECT 'order' FROM pg_catalog.pg_operator WHERE oid IN (${ops.join(", ")}) AND oprname IN ('<', '>', '<=', '>=')`] : []),
+        ...(fns.length > 0 ? [`SELECT 'order' FROM pg_catalog.pg_proc WHERE oid IN (${fns.join(", ")}) AND proname IN ('min', 'max')`] : []),
+        ...(cased.length > 0 ? [`SELECT 'case' FROM pg_catalog.pg_proc WHERE oid IN (${cased.join(", ")}) AND proname IN ('upper', 'lower', 'initcap') AND pronamespace = 'pg_catalog'::regnamespace`] : [])
+    ].join(" UNION ALL "))).map((r) => r[0]);
+    if (named.includes("order"))
+        return "order";
+    // a case mapping alone moves only a statement of text (and the booleans and integers text answers: a comparison, a
+    // length), which the synced data computes as the catalog does but for the case; one that also holds a date, a time,
+    // a json or a session's value stays the catalog's, as before, which answers what the synced data may refuse
+    // (`lower('now')::timestamptz`, a relative time word; Mason gap fromless-numeric-text-order)
+    if (!named.includes("case"))
+        return null;
+    if (![...analysis.types.values()].every((ts) => ts.every((t) => CASE_ROUTED_TYPES.has(t.type))))
+        return null;
+    // and only one whose every value is its own arguments' (review r2 of PR 568, fromless-case-route-run-success-read-as-
+    // sameness): a value of the server or the session (`version()`, `format_type(…)`, `current_database()`,
+    // `current_user`) the synced data answers with ITS OWN, so a run that succeeds says nothing of it. Postgres's analysis
+    // says which: a session keyword or sequence (`functionsCalled`), and a function it does not call immutable
+    // (pg_proc.provolatile: pg_catalog.version() and format_type() are stable); such a statement stays the
+    // catalog's, which answers it as this session's, as before (Mason gap fromless-numeric-text-order)
+    const called = functionsCalled(analysis.tree);
+    if (called.session || called.oids.has(0))
+        return null;
+    const unpure = await catalogRows(o.catalog, `SELECT 1 FROM pg_catalog.pg_proc WHERE oid IN (${[...called.oids].join(", ")}) AND provolatile <> 'i' LIMIT 1`);
+    return unpure.length === 0 ? "case" : null;
+}
+const NUMERIC = 1700, NUMERIC_ARRAY = 1231;
+/** the types a statement of no table that maps case may hold and still be the synced data's: text's, an unknown
+ *  literal's, a name's, a boolean's, an integer's and a text array's */
+const CASE_ROUTED_TYPES = new Set([25, 1043, 1042, 705, 19, 16, 21, 23, 20, 1009, 1015]);
+/** a rewrite's refusal as the statement's: a gap says its id (docs/SQL-SUPPORT.md); Postgres's own refusal is its
+ *  SQLSTATE, and no gap */
+function unrewritableError(e, text) {
+    return typeof e.gap === "string" ? gapRefusal(e.gap, e.message, e.hint, text)
+        : new StatementError({ code: e.gap.postgres, message: e.message, ...(e.hint !== "" ? { hint: e.hint } : {}) }, text);
+}
+/** the functions that read a held value's position, not its value: its type, a count, an array's shape and rows */
+const HELD_READERS = new Set(["1619", "2147", "2803", "3179", "2176", "2091", "2092", "2331", "2334", "730", "2922"]);
+/** What a statement does with a value the copy holds as its text that the text cannot answer as the type would: an
+ *  operator but `=` and `<>` of a type whose equal values have one text, a function, a cast to another type. */
+async function heldRefusal(catalog, tree, text) {
+    const uses = heldUses(tree).filter((u) => !(u.kind === "function" && HELD_READERS.has(u.oid)));
+    if (uses.length === 0)
+        return null;
+    const ops = uses.filter((u) => u.kind === "operator");
+    const equality = new Set(ops.length === 0 ? [] : (await catalogRows(catalog, `SELECT oid::text FROM pg_operator WHERE oid IN (${ops.map((u) => Number(u.oid)).join(", ")}) AND oprname IN ('=', '<>')`)).map((r) => r[0]));
+    const bad = uses.find((u) => !(u.kind === "operator" && equality.has(u.oid) && heldType(u.type)?.textEqual === true && heldType(u.type)?.literal === true && !isHeldArray(u.type)));
+    if (bad === undefined)
+        return null;
+    return gapRefusal("held-type-compare", `Mason cannot answer this over a value of the type ${heldName(bad.type)}: it holds the value's text, and this needs the type's own value`, "read the value, or its text (x::text); = and GROUP BY are answered where equal values have one text", text);
+}
 /** A statement's plan. `wireBound`: its rows are bounded by a row count an Execute sets on the wire, which its SQL
  *  does not say (src/duck/rewrite.ts markBounds; the listener asks for it where PlannedSql.unboundedLateral says so).
  *  `grants`: the session's role (src/duck/grants.ts), held after the analysis; null for a session with no role where
  *  roles exist (the plan still names its reads, for an Execute under a role given later); undefined with no roles at all.
  *  `walls`: `text` is the statement with its filtered reads in place (expandRowFilters), each of which the wall ends */
-export async function planDuck(catalog, text, stmt, base, paramOids, searchPath, timeZone, collation, verified, userTypes = new Map(), copies = () => [], depth = 0, kept = null, wireBound = false, grants = undefined, walls = null) {
+export async function planDuck(options) {
+    const { catalog, text, stmt, base, paramOids, searchPath, timeZone, dateStyle = "ISO, MDY", collations: resolver, verified, userTypes = new Map(), copies = () => [], depth = 0, kept = null, wireBound = false, grants, walls = null, relations, outputs = false } = options;
     const started = performance.now();
-    const analysis = await analyze(catalog, text, paramOids, pathSql(searchPath), timeZone);
+    const under = { searchPath, timeZone, dateStyle };
+    const analysis = await analyze(catalog, text, paramOids, under);
     if ("refused" in analysis) {
         const f = analysis.refused;
         // a table the catalog does not hold: where its copy is, or the `mason add` away
         const missing = f.C === "42P01" ? /^relation "([^"]+)" does not exist$/.exec(f.M ?? "")?.[1] : undefined;
+        // a generated table not built yet (src/duck/generated.ts): by name, and what it is doing
+        const unbuilt = missing === undefined ? undefined : copies().find((c) => c.kept?.by === "generated" && c.phase !== "copied" && (c.source === missing || c.source === `public.${missing}`));
+        if (unbuilt?.kept?.by === "generated")
+            throw gapRefusal("generated-not-built", `the generated table ${unbuilt.source} is not built yet: ${unbuilt.kept.text}`, "mason status says how far it is", text);
         const hint = missing !== undefined ? missingRelationHint(missing, copies()) : f.H;
+        const made = await madeTypeRefusal(catalog, f.C, f.M, text);
+        if (made !== null)
+            throw made;
         throw new StatementError({ code: f.C ?? "XX000", message: f.M ?? "Postgres refused the statement", ...(f.D ? { detail: f.D } : {}), ...(hint ? { hint } : {}),
             ...(f.P ? { position: Number(f.P) } : {}) }, text);
     }
     // A VIEW is answered as Postgres answers it, by its definition in its place (src/duck/views.ts): the statement so
     // written is planned from the start, its parameters typed as this one's were. Its errors are the expanded text's, so
     // they carry no position into this one
-    const expanded = await expandViews(catalog, stmt, analysis, pathSql(searchPath));
+    const expanded = await expandViews(catalog, stmt, analysis, under);
     if (expanded !== null) {
         if (depth >= MAX_VIEW_DEPTH)
             throw viewRefused(`Mason cannot yet answer views read through more than ${MAX_VIEW_DEPTH} other views`, "read the tables the views read");
         try {
-            return await planDuck(catalog, expanded.text, expanded.stmt, 0, analysis.params, searchPath, timeZone, collation, verified, userTypes, copies, depth + 1, kept, wireBound, grants, walls);
+            return await planDuck({ ...options, text: expanded.text, stmt: expanded.stmt, base: 0, paramOids: analysis.params, depth: depth + 1 });
         }
         catch (e) {
             if (!(e instanceof StatementError))
@@ -200,6 +414,9 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
             throw new StatementError(fields, text, e.gap);
         }
     }
+    const held = await heldRefusal(catalog, analysis.tree, text);
+    if (held !== null)
+        throw held;
     // THE ROLE (src/duck/grants.ts): the tables and columns Postgres's analysis says the statement reads, held to the
     // session's role — after Postgres's own refusal and the views' expansion (the check sees the tables a view reads),
     // before the rewrite (a role's refusal before a Mason gap's). The reads stay on the plan: an Execute under a role
@@ -210,11 +427,11 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
     // to nothing more (the grants were held above, on the statement as written: a filter may read a column the role
     // may not, as a policy may on Postgres). Its errors are the expanded text's, so they carry no position into this one
     if (grants?.rows !== undefined && grants.rows.size > 0) {
-        const filtered = await expandRowFilters(catalog, stmt, analysis, pathSql(searchPath), grants.rows);
+        const filtered = await expandRowFilters(catalog, stmt, analysis, under, grants.rows);
         if (filtered !== null) {
             let inner;
             try {
-                inner = await planDuck(catalog, filtered.text, filtered.stmt, 0, analysis.params, searchPath, timeZone, collation, verified, userTypes, copies, depth + 1, kept, wireBound, undefined, filtered.walls);
+                inner = await planDuck({ ...options, text: filtered.text, stmt: filtered.stmt, base: 0, paramOids: analysis.params, depth: depth + 1, grants: undefined, walls: filtered.walls });
             }
             catch (e) {
                 if (!(e instanceof StatementError))
@@ -242,6 +459,9 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
     // the SOURCE's OID for them (`userTypes`, from the copy), never the catalog's
     let resolved = resolvedKeys(analysis.tree);
     const catalogTypes = new Map();
+    // every type that holds a citext anywhere in its structure (`CITEXT_HELD`): the rewrite refuses one where it would
+    // compare it by bytes (rewrite.ts `holdsCitext`)
+    const citextHeld = new Set();
     const enumLabels = new Map(), enumArrays = new Map();
     const enums = { labels: enumLabels, arrays: enumArrays };
     const fields = analysis.fields === null ? null : analysis.fields.map((f) => ({ ...f }));
@@ -252,6 +472,8 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
     if ([...resolved].some((k) => /\bu\d+/.test(k)) || fields?.some((f) => f.oid >= 16384) || used.size > 0) {
         for (const [oid, name] of await catalogRows(catalog, "SELECT oid::text, typname FROM pg_type WHERE oid >= 16384 AND typtype IN ('b', 'd', 'e')"))
             catalogTypes.set(Number(oid), name);
+        for (const [oid] of await catalogRows(catalog, CITEXT_HELD))
+            citextHeld.add(Number(oid));
         // an enum's labels in their declared order, by its OID and its array type's: DuckDB is handed each value as its ENUM
         for (const [oid, array, label] of await catalogRows(catalog, "SELECT t.oid::text, t.typarray::text, e.enumlabel FROM pg_type t JOIN pg_enum e ON e.enumtypid = t.oid ORDER BY t.oid, e.enumsortorder")) {
             const labels = enumLabels.get(Number(oid)) ?? [];
@@ -264,8 +486,8 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
             catalogTypes.set(Number(oid), name);
             if (!used.has(Number(oid)))
                 continue;
-            const columns = await catalogRows(catalog, `SELECT a.attname, a.atttypid::text, a.atttypmod::text FROM pg_attribute a JOIN pg_type t ON t.typrelid = a.attrelid WHERE t.oid = ${Number(oid)} AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum`);
-            rowTypes.set(Number(oid), columns.map(([n, type, typmod]) => ({ name: n, type: Number(type), typmod: Number(typmod) })));
+            const columns = await catalogRows(catalog, `SELECT a.attname, a.atttypid::text, a.atttypmod::text, a.attcollation::text FROM pg_attribute a JOIN pg_type t ON t.typrelid = a.attrelid WHERE t.oid = ${Number(oid)} AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum`);
+            rowTypes.set(Number(oid), columns.map(([n, type, typmod, collation]) => ({ name: n, type: Number(type), typmod: Number(typmod), collation: Number(collation ?? 0) })));
         }
         resolved = await nameUserKeys((sql) => catalogRows(catalog, sql), resolved);
         for (const f of fields ?? []) {
@@ -274,32 +496,45 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
                 f.oid = source;
         }
     }
-    let tree, dropLast, exact, clock, jsonReads, unboundedLateral;
+    let tree, withTies, exact, clock, jsonReads, unboundedLateral;
     // a table's row the source has otherwise (copied in part: no source OID for it) is not the source's row
     const sourceRows = new Map([...rowTypes].filter(([oid]) => userTypes.has(catalogTypes.get(oid) ?? "")));
+    // IS [NOT] NULL of a composite reads its fields (a row of NULLs IS NULL), which the copy's text does not
+    if (used.size > 0) {
+        const rows = await catalogRows(catalog, `SELECT oid::text FROM pg_type WHERE typinput::text LIKE '${HELD_ROW_INPUT}%'`);
+        if (rows.length > 0 && nullTested(analysis.tree, new Set(rows.map((r) => Number(r[0]))))) {
+            throw gapRefusal("held-type-compare", "IS [NOT] NULL of a composite column is not supported by Mason: it holds the value's text, and Postgres reads each of its fields", "read x::text, or the source's own columns", text);
+        }
+    }
     // the stored JSON paths the statement's JSON reads may be answered from (src/duck/kept-route.ts)
     const lookup = kept === null ? null : await keptLookup(catalog, analysis.types, kept);
     const keyCounts = await distinctKeys(catalog, analysis.tree);
+    // WHAT EACH COLLATION OF THE ANALYSIS MEANS (src/duck/collation.ts), resolved before the rewrite reads any: a libc
+    // order not read yet is read here, before the plan is made; a collation the source lacks or Mason cannot follow refused
+    let collations;
+    try {
+        collations = await resolver.prepare((sql) => catalogRows(catalog, sql), collationOids(analysis.tree));
+    }
+    catch (e) {
+        throw e instanceof Unrewritable ? unrewritableError(e, text) : e;
+    }
     const analysed = performance.now();
-    const rewritten = (l = lookup) => rewrite(stmt, base, analysis, collation, catalogTypes, sourceRows, timeZone, enums, l, wireBound, keyCounts);
+    const rewritten = (l = lookup) => rewrite({ stmt: stmt, base, analysis, collations, userTypes: catalogTypes, citextHeld, rowTypes: sourceRows, timeZone, dateStyle, enums, kept: l, wireBound, keyCounts });
     try {
-        ({ tree, dropLast, exact, clock, jsonReads, unboundedLateral } = rewritten());
+        ({ tree, withTies, exact, clock, jsonReads, unboundedLateral } = rewritten());
         // the oracle's (`eager`): every read stored before the plan is made, and the plan made again over them
         if (kept?.keepNow != null && lookup !== null && jsonReads.some((r) => kept.ready(r) === null)) {
             await kept.keepNow(jsonReads);
-            ({ tree, dropLast, exact, clock, jsonReads, unboundedLateral } = rewritten());
+            ({ tree, withTies, exact, clock, jsonReads, unboundedLateral } = rewritten());
         }
         // a statement that lists a copy's columns without naming them reads none from a stored column (hideKept): made
         // again with every read its call, the reads still recorded
         if (lookup !== null && exposes(tree))
-            ({ tree, dropLast, exact, clock, jsonReads, unboundedLateral } = rewritten({ at: (relid, attno) => lookup.at(relid, attno), ready: () => null }));
+            ({ tree, withTies, exact, clock, jsonReads, unboundedLateral } = rewritten({ at: (relid, attno) => lookup.at(relid, attno), ready: () => null }));
     }
     catch (e) {
-        // a gap says its id (docs/SQL-SUPPORT.md); Postgres's own refusal is its SQLSTATE, and no gap
-        if (e instanceof Unrewritable) {
-            throw typeof e.gap === "string" ? gapRefusal(e.gap, e.message, e.hint, text)
-                : new StatementError({ code: e.gap.postgres, message: e.message, hint: e.hint }, text);
-        }
+        if (e instanceof Unrewritable)
+            throw unrewritableError(e, text);
         throw e;
     }
     // a statement that lists a copy's columns without naming them reads its visible ones (src/duck/kept-route.ts); before
@@ -307,7 +542,13 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
     if (kept !== null)
         hideKept(tree, { visible: kept.visible(), searchPath: searchSchemas(searchPath) });
     // a sorted, limited select list is computed over the rows the limit keeps (src/duck/late.ts)
-    tree = selectListAfterLimit(tree, analysis.tree) ?? tree;
+    // (a role's walled read it fetches through itself is one more wall: counted, so `wallStatistics` still walls every one)
+    const lateWall = walls !== null ? { alias: walls.alias, added: 0 } : null;
+    tree = await selectListAfterLimit(tree, analysis.tree, () => joinKeys(catalog, analysis.tree), { volatile: (functions) => volatileFunctions(catalog, functions), wall: lateWall }) ?? tree;
+    // a top N and a min or max under a libc order keyed for their candidates alone (src/duck/libc-bound.ts): after the
+    // late select list, whose inner query holds the sort and the limit; a walled read it reads again is one more wall
+    tree = await libcBounded(tree, analysis.tree, { order: (collate) => resolver.libcOrder(collate),
+        volatile: (functions) => volatileFunctions(catalog, functions), wall: lateWall }) ?? tree;
     // ONLY WHAT THE ORACLE VERIFIED: after the rewrite, so a construct the rewrite refuses by name (with its own way out)
     // says so first
     const keys = [...resolved].sort();
@@ -319,7 +560,11 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
     }
     // only a statement with its filtered reads in place: no other statement's text changes, whatever it names its subqueries
     if (walls !== null && wallSwitch.statistics)
-        wallStatistics(tree, walls);
+        wallStatistics(tree, { alias: walls.alias, reads: walls.reads + (lateWall?.added ?? 0) });
+    // on the finished tree, so no rule above meets it; `masonCalls` below does not count it as a function called
+    giveTurns(tree);
+    if (relations !== undefined && relations.size > 0)
+        swapRelations(tree, relations, searchSchemas(searchPath));
     const sql = printed(tree, text);
     // THE PRINTED TEXT MUST READ BACK AS THE TREE: a printing gap (the deparser drops or respells something) would run
     // another statement than the one Postgres analysed — refused instead, by name
@@ -330,7 +575,7 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
             detail: `read back: ${difference}; the rewritten SQL: ${sql}` }, text);
     }
     const planned = { text: sql, functions: masonCalls(tree), ...(unboundedLateral ? { unboundedLateral: true } : {}) };
-    return { sql: planned, params: analysis.params, fields: fields?.map((f) => ({ ...f, size: pgTyplen(f.oid) })) ?? null, dropLast, ...(exact.length > 0 ? { exact } : {}), resolved: keys,
+    return { sql: planned, params: analysis.params, fields: fields?.map((f) => ({ ...f, size: pgTyplen(f.oid) })) ?? null, withTies, ...(exact.length > 0 ? { exact } : {}), resolved: keys, argumentTypes: argumentTypes(analysis.tree), ...(outputs ? { outputs: outputExpressions(analysis.tree) ?? [] } : {}),
         ...(Object.keys(clock).length > 0 ? { clock } : {}), ...(jsonReads.length > 0 ? { jsonReads: jsonReads.map(readKey) } : {}), ...(reads !== undefined ? { reads } : {}),
         timing: { analysisMs: analysed - started, rewriteMs: performance.now() - analysed } };
 }

package/dist/src/duck/publication.js

@@ -25,6 +25,20 @@ export function namedPublicationVerdict(described, tables, identities) {
     }
     return { refused: null, tables: out, noTruncate };
 }
+/**
+ * A TABLE'S GAP ON ONE LINE, for its line in `mason status` (src/duck/sync.ts `notStreamedReason`): what is wrong, the
+ * statements its owner runs (this module's own "Expected" block, joined), and the way that needs no publication. A gap
+ * written otherwise (one line) is its own first line.
+ */
+export function gapLine(gap) {
+    const lines = gap.split("\n");
+    const at = lines.findIndex((l) => l.startsWith("Expected ("));
+    if (at < 0)
+        return `${lines[0]} — or give its entry sync: poll <interval> in the sources file, which needs no publication`;
+    const end = lines.findIndex((l, i) => i > at && l.trim() === "");
+    const sql = lines.slice(at + 1, end < 0 ? undefined : end).map((l) => l.trim()).filter((l) => !l.startsWith("(")).join(" ");
+    return `${lines[0]} — the database's owner runs ${sql} — or give its entry sync: poll <interval> in the sources file, which needs no publication`;
+}
 // ------------------------------------------------------------------ the check
 const q = (s) => /^[a-z_][a-z0-9_]*$/.test(s) ? s : `"${s.replace(/"/g, '""')}"`;
 const qualified = (t) => t.split(".").map(q).join(".");

package/dist/src/duck/requests.js

@@ -9,7 +9,7 @@ const BATCH_ROWS = 1000;
 /** A request's outcome in a word, from its SQLSTATE (src/api/request-log.ts: the listener's refusal line reads it too). */
 export { outcomeOf };
 const COLUMNS = ["request_id", "started_at", "session", "application_name", "token_name", "user_name", "method", "sqlstate", "message", "detail", "hint",
-    "rows", "duration_ms", "query", "query_truncated", "seq", "json_reads", "bytes_sent", "gap", "role", "attributes"];
+    "rows", "duration_ms", "query", "query_truncated", "seq", "json_reads", "bytes_sent", "gap", "role", "attributes", "client_addr"];
 /** the ledger's DDL: one row per request, named for a developer who knows pg_stat_activity */
 /** a ledger made before `mason user` gains its column; a column added last is named in every INSERT */
 export const REQUESTS_USER_DDL = `ALTER TABLE ${REQUESTS_TABLE} ADD COLUMN IF NOT EXISTS user_name VARCHAR`;
@@ -39,9 +39,13 @@ export const REQUESTS_BYTES_DDL = `ALTER TABLE ${REQUESTS_TABLE} ADD COLUMN IF N
 /** THE REGISTRY ENTRY THAT REFUSED EACH STATEMENT (ApiLogRow.gap, src/duck/gaps.ts): NULL for an answer and for any
  *  other refusal; what `unsupported` groups by. A ledger made before gains it; its old rows are NULL. */
 export const REQUESTS_GAP_DDL = `ALTER TABLE ${REQUESTS_TABLE} ADD COLUMN IF NOT EXISTS gap VARCHAR`;
+/** THE CLIENT'S ADDRESS (ApiLogRow.clientAddress), named as pg_stat_activity names it: behind a TCP proxy the client its
+ *  PROXY header carries, so a deployment behind one can tell who sent what. A ledger made before gains it; its old rows
+ *  are NULL. */
+export const REQUESTS_CLIENT_ADDR_DDL = `ALTER TABLE ${REQUESTS_TABLE} ADD COLUMN IF NOT EXISTS client_addr VARCHAR`;
 export const REQUESTS_DDL = `CREATE TABLE IF NOT EXISTS ${REQUESTS_TABLE} (request_id VARCHAR, started_at TIMESTAMPTZ, session INTEGER, ` +
     `application_name VARCHAR, token_name VARCHAR, user_name VARCHAR, method VARCHAR, sqlstate VARCHAR, message VARCHAR, detail VARCHAR, hint VARCHAR, ` +
-    `"rows" BIGINT, duration_ms INTEGER, query VARCHAR, query_truncated BOOLEAN, seq BIGINT, json_reads VARCHAR, bytes_sent BIGINT, gap VARCHAR, role VARCHAR, attributes VARCHAR)`;
+    `"rows" BIGINT, duration_ms INTEGER, query VARCHAR, query_truncated BOOLEAN, seq BIGINT, json_reads VARCHAR, bytes_sent BIGINT, gap VARCHAR, role VARCHAR, attributes VARCHAR, client_addr VARCHAR)`;
 /** an ApiLogRow's values in COLUMNS' order */
 function valuesOf({ row: r, seq }) {
     const app = (r.client ?? "").replace(/^postgres:/, "");
@@ -49,7 +53,7 @@ function valuesOf({ row: r, seq }) {
         r.sqlstate ?? null, r.error === undefined || r.error === "" ? null : r.error, r.detail ?? null, r.hint ?? null,
         r.rows === undefined ? null : r.rows, Math.round(r.durationMs), r.payload ?? "", r.payloadTruncated === true, seq,
         r.jsonReads === undefined || r.jsonReads.length === 0 ? null : JSON.stringify(r.jsonReads), r.bytesSent === undefined ? null : r.bytesSent, r.gap ?? null, r.role ?? "",
-        r.attributes === undefined || Object.keys(r.attributes).length === 0 ? null : JSON.stringify(r.attributes)];
+        r.attributes === undefined || Object.keys(r.attributes).length === 0 ? null : JSON.stringify(r.attributes), r.clientAddress ?? null];
 }
 /** A ledger row as the commands read it. */
 function requestOf(o) {
@@ -59,7 +63,7 @@ function requestOf(o) {
     const startedAt = new Date(String(o.started_at));
     return {
         requestId: String(o.request_id), startedAt: startedAt.toISOString(), session: o.session === null ? null : Number(o.session),
-        applicationName: String(o.application_name ?? ""), tokenName: String(o.token_name ?? ""), user: String(o.user_name ?? ""), role: String(o.role ?? ""),
+        applicationName: String(o.application_name ?? ""), clientAddress: typeof o.client_addr === "string" ? o.client_addr : null, tokenName: String(o.token_name ?? ""), user: String(o.user_name ?? ""), role: String(o.role ?? ""),
         attributes: typeof o.attributes === "string" && o.attributes !== "" ? JSON.parse(o.attributes) : null, method: String(o.method),
         outcome, status: statusOf(outcome), sqlstate, message: text(o.message), detail: text(o.detail), hint: text(o.hint),
         rows: o.rows === null ? null : Number(o.rows), durationMs: Number(o.duration_ms),
@@ -131,7 +135,7 @@ export class RequestLedger {
     }
     /** The ledger of a store: its table made, the rows past the retention deleted, and a trim every hour. */
     static async open(store, log, options) {
-        const history = await History.open(store, [REQUESTS_DDL, REQUESTS_USER_DDL, REQUESTS_SEQ_DDL, REQUESTS_JSON_DDL, REQUESTS_BYTES_DDL, REQUESTS_GAP_DDL, REQUESTS_ROLE_DDL, REQUESTS_ATTRIBUTES_DDL]);
+        const history = await History.open(store, [REQUESTS_DDL, REQUESTS_USER_DDL, REQUESTS_SEQ_DDL, REQUESTS_JSON_DDL, REQUESTS_BYTES_DDL, REQUESTS_GAP_DDL, REQUESTS_ROLE_DDL, REQUESTS_ATTRIBUTES_DDL, REQUESTS_CLIENT_ADDR_DDL]);
         const ledger = new RequestLedger(history, log, options);
         ledger.next = Number((await history.reader.runAndReadAll(`SELECT coalesce(max(seq), -1) + 1 FROM ${REQUESTS_TABLE}`)).getRowsJS()[0][0]);
         await history.keepTrimmed(() => ledger.trim(), (e) => log(`request log: deleting the requests older than ${ledger.days} days failed: ${e.message}`));