@mason-data/cli 0.7.0 → 0.9.0

package/dist/src/duck/plan.js

@@ -6,17 +6,22 @@
  * per text until the schema changes.
  */
 import { deparseSync, QuoteUtils } from "pgsql-deparser";
-import { StatementError } from "../api/postgres/server.js";
+import { HELD_ROW_INPUT, HELD_TYPE_INPUT } from "../api/postgres/schema.js";
+import { heldName, heldType, isHeldArray } from "./held-types.js";
+import { collationOids, functionsCalled, outputExpressions, textOrdering } from "./pgtree.js";
+import { gapRefusal, StatementError } from "./statement.js";
 import { parseSql } from "../api/postgres/parser.js";
 import { analyze, catalogRows, readParameters } from "./analyze.js";
-import { describeKeys, nameUserKeys, resolvedKeys } from "./resolved.js";
-import { gapMark } from "./gaps.js";
+import { argumentTypes, describeKeys, nameUserKeys, heldUses, nullTested, resolvedKeys } from "./resolved.js";
 import { expandViews, MAX_VIEW_DEPTH, viewRefused } from "./views.js";
 import { selectListAfterLimit } from "./late.js";
+import { libcBounded } from "./libc-bound.js";
 import { pgTyplen } from "./types.js";
-import { datumText, masonCalls, rewrite, treeDifference, Unrewritable } from "./rewrite.js";
+import { giveTurns, datumText, masonCalls, rewrite, treeDifference, Unrewritable } from "./rewrite.js";
 import { exposes, hideKept, readKey, searchSchemas } from "./kept-route.js";
-import { distinctCountTables, distinctKeyCounts, uniqueColumnsOf, uniqueColumnsSql } from "./distinct-key.js";
+import { distinctCountTables, distinctKeyCounts, fanOutTables, groupedLimitTables, primaryKeysOf, uniqueColumnsOf, uniqueColumnsSql } from "./distinct-key.js";
+import { enforceGrants } from "./grants.js";
+import { attributeRefused, expandRowFilters, wallStatistics, wallSwitch } from "./row-filter.js";
 const JSON_OID = 114, JSONB_OID = 3802;
 /** what the rewrite asks of the stored JSON paths (src/duck/kept-route.ts KeptLookup): the copied table and column of
  *  each json or jsonb column the statement reads from a table (Typed.relation), by the catalog's OIDs; null when it
@@ -43,6 +48,20 @@ async function distinctKeys(catalog, tree) {
         return new Set();
     return distinctKeyCounts(tree, uniqueColumnsOf(await catalogRows(catalog, uniqueColumnsSql(relids))));
 }
+/** the primary keys that may prove a LIMIT with no ORDER BY's joins bounded, or a grouped LIMIT grouped by a whole key
+ *  (src/duck/late.ts), read only for a statement that has one */
+async function joinKeys(catalog, tree) {
+    const relids = [...new Set([...fanOutTables(tree), ...groupedLimitTables(tree)])].sort((a, b) => a - b);
+    return relids.length === 0 ? new Map() : primaryKeysOf(await catalogRows(catalog, uniqueColumnsSql(relids)));
+}
+/** which of `functions` the catalog says are volatile: asked only of a statement with a derived read the list after the
+ *  limit may read twice (src/duck/late.ts) */
+async function volatileFunctions(catalog, functions) {
+    if (functions.length === 0)
+        return new Set();
+    const rows = await catalogRows(catalog, `SELECT oid::text FROM pg_proc WHERE provolatile = 'v' AND oid IN (${functions.map((f) => String(Math.trunc(f))).join(", ")})`);
+    return new Set(rows.map(([oid]) => Number(oid)));
+}
 /**
  * DuckDB's keywords that Postgres's are not: a column named `at` (DuckDB's AT TIME ZONE) or `qualify` is an identifier
  * the deparser writes bare, as Postgres reads it, and DuckDB refuses as a syntax error. The deparser's one quoting
@@ -59,9 +78,17 @@ export function quoteDuckKeywords(words) {
         return q === ident && duckKeywords.has(ident) ? `"${ident}"` : q;
     };
 }
-/** a search_path as SET writes it: each schema quoted (`"$user"` kept) */
-const pathSql = (searchPath) => searchPath.split(",").map((s) => s.trim()).filter((s) => s !== "")
-    .map((s) => s.startsWith('"') ? s : `"${s.replaceAll('"', '""')}"`).join(", ") || "public";
+/**
+ * EVERY TYPE THAT HOLDS A CITEXT, to any depth, by the catalog's structure: citext itself; an array by its element
+ * (`typelem`, a nested array is the same type); a domain by its base (`typbasetype`); a composite by its attributes; a
+ * range by its subtype and a multirange by its range (PR 674 shrink 2: an array of citext compared by bytes in a
+ * quantified subquery). One recursive term, since Postgres takes the recursive reference once.
+ */
+export const CITEXT_HELD = "WITH RECURSIVE held(oid) AS (SELECT oid FROM pg_type WHERE typname = 'citext' UNION " +
+    "SELECT t.oid FROM pg_type t, held h WHERE t.typelem = h.oid OR t.typbasetype = h.oid " +
+    "OR (t.typrelid <> 0 AND EXISTS (SELECT 1 FROM pg_attribute a WHERE a.attrelid = t.typrelid AND a.attnum > 0 AND NOT a.attisdropped AND a.atttypid = h.oid)) " +
+    "OR EXISTS (SELECT 1 FROM pg_range r WHERE (r.rngtypid = t.oid AND r.rngsubtype = h.oid) OR (r.rngmultitypid = t.oid AND r.rngtypid = h.oid))) " +
+    "SELECT oid::text FROM held";
 /**
  * `verified`: what the oracle has held to Postgres (src/duck/verified.ts). A statement using anything else is REFUSED,
  * by name — never answered by a rewrite nobody checked. null only for the oracle itself, which records what it verifies.
@@ -71,12 +98,18 @@ const pathSql = (searchPath) => searchPath.split(",").map((s) => s.trim()).filte
  * src/duck/analyze.ts), written back in a form DuckDB reads to the same value. A value Postgres refuses is refused with
  * its error (`'1.5'` for a smallint: 22P02).
  */
-export async function duckParameters(catalog, oids, values, timeZone) {
-    const read = await readParameters(catalog, oids, values, timeZone);
+export async function duckParameters(catalog, oids, values, under) {
+    const read = await readParameters(catalog, oids, values, under);
     if ("refused" in read) {
         const f = read.refused;
         throw new StatementError({ code: f.C ?? "22P02", message: f.M ?? "invalid parameter value", ...(f.D ? { detail: f.D } : {}), ...(f.H ? { hint: f.H } : {}) }, "");
     }
+    // a text[] value DuckDB's list cannot hold as Postgres read it: refused by name, as its literal is
+    if ("gap" in read) {
+        throw read.gap === "multidim-array"
+            ? gapRefusal("multidim-array", `a multi-dimensional array as parameter $${read.parameter} is not supported by Mason`, "bind a one-dimensional array")
+            : gapRefusal("array-lower-bound", `an array whose subscripts do not start at 1 as parameter $${read.parameter} is not supported by Mason`, "bind the array without its bounds ('{a,b}', not '[2:3]={a,b}')");
+    }
     return values.map((v, i) => {
         const r = read.values[i];
         if (v === null || r === undefined || r === null)
@@ -106,11 +139,11 @@ export function missingRelationHint(relation, copies) {
     const matches = copies.filter((c) => c.source === relation || (!relation.includes(".") && c.source.endsWith(`.${relation}`)))
         .sort((a, b) => Number(b.source.startsWith("public.")) - Number(a.source.startsWith("public.")));
     const c = matches[0];
-    const served = "it is served once its copy commits; mason wait waits for it (mason status shows how far it is)";
+    const served = "it is served once its first sync commits; mason wait waits for it (mason status shows how far it is)";
     if (c === undefined)
-        return `a table is in Mason once it is added: mason add ${relation} copies it (mason tables lists the database's)`;
+        return `a table is in Mason once it is added: mason add ${relation} syncs it (mason tables lists the database's)`;
     switch (c.phase) {
-        case "waiting": return `${c.source} is added and waits its turn to be copied: ${served}`;
+        case "waiting": return `${c.source} is added and waits its turn to be synced: ${served}`;
         case "copying": {
             // the share is of the heap's PAGES read (`CopyProgressView.share`: exact, so it never moves backwards), the count
             // the rows the committed chunks and the ones reading have in; a copy can outrun the source's estimate (Postgres's
@@ -123,46 +156,312 @@ export function missingRelationHint(relation, copies) {
                 : of === null || of <= 0 ? `${share}about ${so} rows so far`
                     : so > of ? `${share}${so} rows so far, past the ~${of} estimated`
                         : `${share}about ${so} of ~${of} rows so far${p === undefined ? `, ${Math.min(99, Math.floor((so / of) * 100))}%` : ""}`;
-            return `${c.source} is added and being copied (${how}): ${served}`;
+            return `${c.source} is added and being synced (${how}): ${served}`;
+        }
+        case "failed": return `${c.source} is added, but its first sync failed: ${c.error ?? "no reason recorded"}; mason logs says more`;
+        case "copied": return `${c.source} was just synced and joins the catalog as it is rebuilt: run the statement again in a moment (if it stays missing, mason logs says why)`;
+    }
+}
+/**
+ * THE TREE AS TEXT, DuckDB's `COLUMNS(…)` (decidedByKey's star, the statistics wall) unquoted: it is an expression only so,
+ * and the deparser quotes the keyword. Marked in the tree, never found in the printed text, where a literal or a name may
+ * hold the same characters: each call printed under a name the text holds nowhere else (the count says so), then that name
+ * replaced. No statement Postgres accepted calls a function "columns" (Postgres has none), and the read-back checks the text.
+ */
+function printed(tree, statement) {
+    const calls = [];
+    const collect = (v) => {
+        if (Array.isArray(v)) {
+            v.forEach(collect);
+            return;
+        }
+        if (typeof v !== "object" || v === null)
+            return;
+        const name = v.FuncCall?.funcname;
+        if (name?.length === 1 && name[0]?.String?.sval === "columns")
+            calls.push(name[0]);
+        Object.values(v).forEach(collect);
+    };
+    collect(tree);
+    if (calls.length === 0)
+        return deparseSync(tree, { pretty: false });
+    // a text holds a mark at most once per few characters; a count that never agrees is a call the deparser does not print
+    const bound = deparseSync(tree, { pretty: false }).length;
+    for (let i = 0; i <= bound; i++) {
+        const mark = `__mason_columns_${i}`;
+        for (const c of calls)
+            c.String.sval = mark;
+        const text = deparseSync(tree, { pretty: false });
+        for (const c of calls)
+            c.String.sval = "columns";
+        if (text.split(`${mark}(`).length - 1 === calls.length)
+            return text.replaceAll(`${mark}(`, "COLUMNS(");
+    }
+    throw new StatementError({ code: "0A000", message: "Mason cannot print this statement for the engine (a Mason gap: report it, with the statement)" }, statement);
+}
+/**
+ * A CHAIN'S CHILD READS ITS PARENT'S SIDE TABLE (src/duck/generated.ts, issue 124 stage 2): on the finished tree, each
+ * read of a table `relations` names is swapped for the table it maps to, in the same schema, under the name the statement
+ * read it by (its alias, else the table's own), so every column the rewrite qualified still binds. A bare name is the
+ * search path's first schema's (a generated table is planned under `public`), and a bare name a WITH in scope holds is a
+ * CTE's, never swapped. Every table named must be met: a read left on the served table would read the old parent, so
+ * one not met throws. The read-back below holds the swapped text to the tree, as any other.
+ */
+function swapRelations(tree, relations, schemas) {
+    const met = new Set();
+    const walk = (v, ctes) => {
+        if (Array.isArray(v)) {
+            for (const x of v)
+                walk(x, ctes);
+            return;
+        }
+        if (typeof v !== "object" || v === null)
+            return;
+        const o = v;
+        const r = o.RangeVar;
+        if (r !== undefined) {
+            if (r.schemaname === undefined && ctes.has(r.relname))
+                return;
+            const key = `${r.schemaname ?? schemas[0] ?? "public"}.${r.relname}`;
+            const to = relations.get(key);
+            if (to !== undefined) {
+                r.alias ??= { aliasname: r.relname };
+                r.relname = to;
+                met.add(key);
+            }
+            return;
         }
-        case "failed": return `${c.source} is added, but its copy failed: ${c.error ?? "no reason recorded"}; mason logs says more`;
-        case "copied": return `${c.source} was just copied and joins the catalog as it is rebuilt: run the statement again in a moment (if it stays missing, mason logs says why)`;
+        // a column by the table's schema-qualified name (`public.counts.tenant_id`, valid Postgres): the swapped table is
+        // read under its own name as an alias, which binds no schema, so the column is named by that alias alone
+        const fields = o.ColumnRef?.fields;
+        if (fields !== undefined && fields.length >= 3) {
+            const sval = (x) => x?.String?.sval;
+            const schema = sval(fields[fields.length - 3]), table = sval(fields[fields.length - 2]);
+            if (schema !== undefined && table !== undefined && relations.has(`${schema}.${table}`))
+                o.ColumnRef.fields = fields.slice(-2);
+            return;
+        }
+        const list = o.withClause?.ctes;
+        const scope = list === undefined ? ctes : new Set([...ctes, ...list.map((c) => c.CommonTableExpr?.ctename ?? "")]);
+        for (const x of Object.values(o))
+            walk(x, scope);
+    };
+    walk(tree, new Set());
+    const missed = [...relations.keys()].filter((k) => !met.has(k));
+    if (missed.length > 0)
+        throw new Error(`a generated table's plan does not read ${missed.join(", ")} by a name it can swap (Mason bug: report it)`);
+}
+/** The catalog declares a type the source made with nothing over it (`PgColumn.made`), so what its analysis says is not
+ *  there (an `=`, an order, a field of a composite) the source may have: Mason's gap, never Postgres's refusal. */
+async function madeTypeRefusal(catalog, code, message, text) {
+    if (message === undefined || (code !== "42883" && code !== "42809" && code !== "42804" && code !== "42846"))
+        return null;
+    const types = (await catalogRows(catalog, `SELECT t.typname FROM pg_type t WHERE t.typinput::text LIKE '%${HELD_TYPE_INPUT}%'`)).map((r) => r[0]);
+    const name = types.find((t) => new RegExp(`\\b${t.replace(/[^\w]/g, ".")}\\b`).test(message));
+    if (name === undefined || /^function /.test(message) && !/\(.*\)/.test(message))
+        return null;
+    return gapRefusal("held-type-compare", `Mason cannot answer this over a column of the type ${name}: it holds the value's text, and has no operator of the type`, "read the value, or its text (x::text); compare, sort or take a field on the source's own columns", text);
+}
+/**
+ * A STATEMENT OF NO TABLE THAT ORDERS TEXT UNDER A COLLATION THAT IS NOT BYTE ORDER (issue 122): the catalog's database
+ * is C, so `SELECT 'a' < 'B'` answered there is byte order where the source's libc or ICU says otherwise. Its analysis
+ * (under the session's state) says where it orders a collatable value and under which collation (`textOrdering`); an
+ * operator or a function counts as ordering by the catalog's own name of it (`<`, `max` …, never `=` or `count`), and
+ * the resolver says whether that collation is byte order. "order": the statement is planned for the synced data, made
+ * and refused like any other. A statement the analysis refuses stays the catalog's, which raises Postgres's own error.
+ * "case" (issue 406): it calls upper, lower or initcap of text, under any collation (the catalog maps case by its own
+ * C.UTF-8's rules, never the source's ctype): the synced data's where its RUN answers the statement, which the caller
+ * decides (src/api/postgres/server.ts `answersCase`), since what the catalog answered before is kept where the synced
+ * data's plan or run fails.
+ */
+export async function ordersText(o) {
+    const analysis = await analyze(o.catalog, o.text, o.paramOids, o.under);
+    if ("refused" in analysis)
+        return null;
+    // the analysis as the session runs the statement: a simple Query binds no parameter, so one naming `$1` is Postgres's
+    // 42P02, which the catalog answers (the analysis above is a Parse's, which would infer its type)
+    if (!o.bindable && analysis.params.length > 0)
+        return null;
+    const oids = collationOids(analysis.tree);
+    if (oids.size === 0)
+        return null;
+    let resolved;
+    // (a collation the source lacks, or one Mason cannot follow: the synced data's plan refuses it by name, never the
+    // catalog's byte order; a nondeterministic one wherever it is named, equality and GROUP BY included)
+    try {
+        resolved = await o.collations.prepare((sql) => catalogRows(o.catalog, sql), oids);
     }
+    catch (e) {
+        if (e instanceof Unrewritable)
+            return "order";
+        throw e;
+    }
+    // a statement of no table that holds a numeric (a literal such as 1.5 included) stays the catalog's, which computes it exactly, its text in byte
+    // order and its case as before (Mason gap fromless-numeric-text-order): the synced data holds a VALUES list's numeric
+    // otherwise than Postgres (issue 342's class), and the catalog answered it right
+    if ([...analysis.types.values()].some((ts) => ts.some((t) => t.type === NUMERIC || t.type === NUMERIC_ARRAY)))
+        return null;
+    const ordering = (coll) => {
+        try {
+            const c = resolved.resolve(coll);
+            return c !== null && c.kind !== "bytes";
+        }
+        catch (e) {
+            if (e instanceof Unrewritable)
+                return true;
+            throw e;
+        }
+    };
+    const { orders, operators, functions, calls } = textOrdering(analysis.tree);
+    if ([...orders].some(ordering))
+        return "order";
+    const ops = [...operators].filter(([, cs]) => [...cs].some(ordering)).map(([oid]) => oid);
+    const fns = [...functions].filter(([, cs]) => [...cs].some(ordering)).map(([oid]) => oid);
+    const cased = [...calls.keys()];
+    if (ops.length + fns.length + cased.length === 0)
+        return null;
+    const named = (await catalogRows(o.catalog, [
+        ...(ops.length > 0 ? [`SELECT 'order' FROM pg_catalog.pg_operator WHERE oid IN (${ops.join(", ")}) AND oprname IN ('<', '>', '<=', '>=')`] : []),
+        ...(fns.length > 0 ? [`SELECT 'order' FROM pg_catalog.pg_proc WHERE oid IN (${fns.join(", ")}) AND proname IN ('min', 'max')`] : []),
+        ...(cased.length > 0 ? [`SELECT 'case' FROM pg_catalog.pg_proc WHERE oid IN (${cased.join(", ")}) AND proname IN ('upper', 'lower', 'initcap') AND pronamespace = 'pg_catalog'::regnamespace`] : [])
+    ].join(" UNION ALL "))).map((r) => r[0]);
+    if (named.includes("order"))
+        return "order";
+    // a case mapping alone moves only a statement of text (and the booleans and integers text answers: a comparison, a
+    // length), which the synced data computes as the catalog does but for the case; one that also holds a date, a time,
+    // a json or a session's value stays the catalog's, as before, which answers what the synced data may refuse
+    // (`lower('now')::timestamptz`, a relative time word; Mason gap fromless-numeric-text-order)
+    if (!named.includes("case"))
+        return null;
+    if (![...analysis.types.values()].every((ts) => ts.every((t) => CASE_ROUTED_TYPES.has(t.type))))
+        return null;
+    // and only one whose every value is its own arguments' (review r2 of PR 568, fromless-case-route-run-success-read-as-
+    // sameness): a value of the server or the session (`version()`, `format_type(…)`, `current_database()`,
+    // `current_user`) the synced data answers with ITS OWN, so a run that succeeds says nothing of it. Postgres's analysis
+    // says which: a session keyword or sequence (`functionsCalled`), and a function it does not call immutable
+    // (pg_proc.provolatile: pg_catalog.version() and format_type() are stable); such a statement stays the
+    // catalog's, which answers it as this session's, as before (Mason gap fromless-numeric-text-order)
+    const called = functionsCalled(analysis.tree);
+    if (called.session || called.oids.has(0))
+        return null;
+    const unpure = await catalogRows(o.catalog, `SELECT 1 FROM pg_catalog.pg_proc WHERE oid IN (${[...called.oids].join(", ")}) AND provolatile <> 'i' LIMIT 1`);
+    return unpure.length === 0 ? "case" : null;
+}
+const NUMERIC = 1700, NUMERIC_ARRAY = 1231;
+/** the types a statement of no table that maps case may hold and still be the synced data's: text's, an unknown
+ *  literal's, a name's, a boolean's, an integer's and a text array's */
+const CASE_ROUTED_TYPES = new Set([25, 1043, 1042, 705, 19, 16, 21, 23, 20, 1009, 1015]);
+/** a rewrite's refusal as the statement's: a gap says its id (docs/SQL-SUPPORT.md); Postgres's own refusal is its
+ *  SQLSTATE, and no gap */
+function unrewritableError(e, text) {
+    return typeof e.gap === "string" ? gapRefusal(e.gap, e.message, e.hint, text)
+        : new StatementError({ code: e.gap.postgres, message: e.message, ...(e.hint !== "" ? { hint: e.hint } : {}) }, text);
+}
+/** the functions that read a held value's position, not its value: its type, a count, an array's shape and rows */
+const HELD_READERS = new Set(["1619", "2147", "2803", "3179", "2176", "2091", "2092", "2331", "2334", "730", "2922"]);
+/** What a statement does with a value the copy holds as its text that the text cannot answer as the type would: an
+ *  operator but `=` and `<>` of a type whose equal values have one text, a function, a cast to another type. */
+async function heldRefusal(catalog, tree, text) {
+    const uses = heldUses(tree).filter((u) => !(u.kind === "function" && HELD_READERS.has(u.oid)));
+    if (uses.length === 0)
+        return null;
+    const ops = uses.filter((u) => u.kind === "operator");
+    const equality = new Set(ops.length === 0 ? [] : (await catalogRows(catalog, `SELECT oid::text FROM pg_operator WHERE oid IN (${ops.map((u) => Number(u.oid)).join(", ")}) AND oprname IN ('=', '<>')`)).map((r) => r[0]));
+    const bad = uses.find((u) => !(u.kind === "operator" && equality.has(u.oid) && heldType(u.type)?.textEqual === true && heldType(u.type)?.literal === true && !isHeldArray(u.type)));
+    if (bad === undefined)
+        return null;
+    return gapRefusal("held-type-compare", `Mason cannot answer this over a value of the type ${heldName(bad.type)}: it holds the value's text, and this needs the type's own value`, "read the value, or its text (x::text); = and GROUP BY are answered where equal values have one text", text);
 }
 /** A statement's plan. `wireBound`: its rows are bounded by a row count an Execute sets on the wire, which its SQL
- *  does not say (src/duck/rewrite.ts markBounds; the listener asks for it where PlannedSql.unboundedLateral says so) */
-export async function planDuck(catalog, text, stmt, base, paramOids, searchPath, timeZone, collation, verified, userTypes = new Map(), copies = () => [], depth = 0, kept = null, wireBound = false) {
+ *  does not say (src/duck/rewrite.ts markBounds; the listener asks for it where PlannedSql.unboundedLateral says so).
+ *  `grants`: the session's role (src/duck/grants.ts), held after the analysis; null for a session with no role where
+ *  roles exist (the plan still names its reads, for an Execute under a role given later); undefined with no roles at all.
+ *  `walls`: `text` is the statement with its filtered reads in place (expandRowFilters), each of which the wall ends */
+export async function planDuck(options) {
+    const { catalog, text, stmt, base, paramOids, searchPath, timeZone, dateStyle = "ISO, MDY", collations: resolver, verified, userTypes = new Map(), copies = () => [], depth = 0, kept = null, wireBound = false, grants, walls = null, relations, outputs = false } = options;
     const started = performance.now();
-    const analysis = await analyze(catalog, text, paramOids, pathSql(searchPath), timeZone);
+    const under = { searchPath, timeZone, dateStyle };
+    const analysis = await analyze(catalog, text, paramOids, under);
     if ("refused" in analysis) {
         const f = analysis.refused;
         // a table the catalog does not hold: where its copy is, or the `mason add` away
         const missing = f.C === "42P01" ? /^relation "([^"]+)" does not exist$/.exec(f.M ?? "")?.[1] : undefined;
+        // a generated table not built yet (src/duck/generated.ts): by name, and what it is doing
+        const unbuilt = missing === undefined ? undefined : copies().find((c) => c.kept?.by === "generated" && c.phase !== "copied" && (c.source === missing || c.source === `public.${missing}`));
+        if (unbuilt?.kept?.by === "generated")
+            throw gapRefusal("generated-not-built", `the generated table ${unbuilt.source} is not built yet: ${unbuilt.kept.text}`, "mason status says how far it is", text);
         const hint = missing !== undefined ? missingRelationHint(missing, copies()) : f.H;
+        const made = await madeTypeRefusal(catalog, f.C, f.M, text);
+        if (made !== null)
+            throw made;
         throw new StatementError({ code: f.C ?? "XX000", message: f.M ?? "Postgres refused the statement", ...(f.D ? { detail: f.D } : {}), ...(hint ? { hint } : {}),
             ...(f.P ? { position: Number(f.P) } : {}) }, text);
     }
     // A VIEW is answered as Postgres answers it, by its definition in its place (src/duck/views.ts): the statement so
     // written is planned from the start, its parameters typed as this one's were. Its errors are the expanded text's, so
     // they carry no position into this one
-    const expanded = await expandViews(catalog, stmt, analysis, pathSql(searchPath));
+    const expanded = await expandViews(catalog, stmt, analysis, under);
     if (expanded !== null) {
         if (depth >= MAX_VIEW_DEPTH)
             throw viewRefused(`Mason cannot yet answer views read through more than ${MAX_VIEW_DEPTH} other views`, "read the tables the views read");
         try {
-            return await planDuck(catalog, expanded.text, expanded.stmt, 0, analysis.params, searchPath, timeZone, collation, verified, userTypes, copies, depth + 1, kept, wireBound);
+            return await planDuck({ ...options, text: expanded.text, stmt: expanded.stmt, base: 0, paramOids: analysis.params, depth: depth + 1 });
         }
         catch (e) {
             if (!(e instanceof StatementError))
                 throw e;
             const { position: _p, ...fields } = e.fields;
-            throw new StatementError(fields, text);
+            throw new StatementError(fields, text, e.gap);
+        }
+    }
+    const held = await heldRefusal(catalog, analysis.tree, text);
+    if (held !== null)
+        throw held;
+    // THE ROLE (src/duck/grants.ts): the tables and columns Postgres's analysis says the statement reads, held to the
+    // session's role — after Postgres's own refusal and the views' expansion (the check sees the tables a view reads),
+    // before the rewrite (a role's refusal before a Mason gap's). The reads stay on the plan: an Execute under a role
+    // given or taken since the Parse is held to them again (src/api/postgres/server.ts)
+    const reads = grants === undefined ? undefined : await enforceGrants(catalog, analysis.tree, grants);
+    // THE ROLE'S ROWS (src/duck/row-filter.ts): every read of a table the role filters becomes its filtered read, and that
+    // statement is planned from the start — analysed by Postgres, rewritten and verified as any other (one grammar), held
+    // to nothing more (the grants were held above, on the statement as written: a filter may read a column the role
+    // may not, as a policy may on Postgres). Its errors are the expanded text's, so they carry no position into this one
+    if (grants?.rows !== undefined && grants.rows.size > 0) {
+        const filtered = await expandRowFilters(catalog, stmt, analysis, under, grants.rows);
+        if (filtered !== null) {
+            let inner;
+            try {
+                inner = await planDuck({ ...options, text: filtered.text, stmt: filtered.stmt, base: 0, paramOids: analysis.params, depth: depth + 1, grants: undefined, walls: filtered.walls });
+            }
+            catch (e) {
+                if (!(e instanceof StatementError))
+                    throw e;
+                // a data exception may be a bound value's, not the statement's: `--set tenant=abc` on an integer
+                const bad = e.fields.code.startsWith("22") && e.gap === null ? await attributeRefused(catalog, grants.rows, filtered.tables) : null;
+                if (bad !== null) {
+                    const user = JSON.stringify(grants.rowsUser ?? "");
+                    throw new StatementError({ code: "42501", message: `permission denied: the user ${user} has a value for the attribute ${bad.name} that is not of type ${bad.type}, which the role ${grants.name} filters rows by`,
+                        detail: `${bad.at} reads :${bad.name} as ${bad.type} (${bad.where}): ${bad.problem}`,
+                        hint: `mason user remove ${grants.rowsUser ?? "<user>"} && mason user add ${grants.rowsUser ?? "<user>"} --role ${grants.name} --set ${bad.name}=<a value of type ${bad.type}>` }, text);
+                }
+                // the statement as written passed this check above, so what fails it is the table being a subquery (carryKeyDependency)
+                if (e.fields.code === "42803" || /could not identify an equality operator/.test(e.fields.message)) {
+                    throw gapRefusal("roles", `Mason cannot yet answer a GROUP BY of a key of a table its role filters here: ${e.fields.message}`, "name each column it reads in the GROUP BY, or read it in an aggregate", text);
+                }
+                const { position: _p, ...fields } = e.fields;
+                throw new StatementError(fields, text, e.gap);
+            }
+            const rowFilters = filtered.tables.map((t) => ({ table: t, words: `filtered by the role ${grants.name}: ${grants.rows.get(t).words}` }));
+            return { ...inner, ...(reads !== undefined ? { reads } : {}), rowFilters };
         }
     }
     // the catalog's own OIDs for the types an extension made (citext): the rewrite knows a few by name, and the wire sends
     // the SOURCE's OID for them (`userTypes`, from the copy), never the catalog's
     let resolved = resolvedKeys(analysis.tree);
     const catalogTypes = new Map();
+    // every type that holds a citext anywhere in its structure (`CITEXT_HELD`): the rewrite refuses one where it would
+    // compare it by bytes (rewrite.ts `holdsCitext`)
+    const citextHeld = new Set();
     const enumLabels = new Map(), enumArrays = new Map();
     const enums = { labels: enumLabels, arrays: enumArrays };
     const fields = analysis.fields === null ? null : analysis.fields.map((f) => ({ ...f }));
@@ -173,6 +472,8 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
     if ([...resolved].some((k) => /\bu\d+/.test(k)) || fields?.some((f) => f.oid >= 16384) || used.size > 0) {
         for (const [oid, name] of await catalogRows(catalog, "SELECT oid::text, typname FROM pg_type WHERE oid >= 16384 AND typtype IN ('b', 'd', 'e')"))
             catalogTypes.set(Number(oid), name);
+        for (const [oid] of await catalogRows(catalog, CITEXT_HELD))
+            citextHeld.add(Number(oid));
         // an enum's labels in their declared order, by its OID and its array type's: DuckDB is handed each value as its ENUM
         for (const [oid, array, label] of await catalogRows(catalog, "SELECT t.oid::text, t.typarray::text, e.enumlabel FROM pg_type t JOIN pg_enum e ON e.enumtypid = t.oid ORDER BY t.oid, e.enumsortorder")) {
             const labels = enumLabels.get(Number(oid)) ?? [];
@@ -185,8 +486,8 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
             catalogTypes.set(Number(oid), name);
             if (!used.has(Number(oid)))
                 continue;
-            const columns = await catalogRows(catalog, `SELECT a.attname, a.atttypid::text, a.atttypmod::text FROM pg_attribute a JOIN pg_type t ON t.typrelid = a.attrelid WHERE t.oid = ${Number(oid)} AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum`);
-            rowTypes.set(Number(oid), columns.map(([n, type, typmod]) => ({ name: n, type: Number(type), typmod: Number(typmod) })));
+            const columns = await catalogRows(catalog, `SELECT a.attname, a.atttypid::text, a.atttypmod::text, a.attcollation::text FROM pg_attribute a JOIN pg_type t ON t.typrelid = a.attrelid WHERE t.oid = ${Number(oid)} AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum`);
+            rowTypes.set(Number(oid), columns.map(([n, type, typmod, collation]) => ({ name: n, type: Number(type), typmod: Number(typmod), collation: Number(collation ?? 0) })));
         }
         resolved = await nameUserKeys((sql) => catalogRows(catalog, sql), resolved);
         for (const f of fields ?? []) {
@@ -195,33 +496,45 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
                 f.oid = source;
         }
     }
-    let tree, dropLast, exact, clock, jsonReads, unboundedLateral;
+    let tree, withTies, exact, clock, jsonReads, unboundedLateral;
     // a table's row the source has otherwise (copied in part: no source OID for it) is not the source's row
     const sourceRows = new Map([...rowTypes].filter(([oid]) => userTypes.has(catalogTypes.get(oid) ?? "")));
+    // IS [NOT] NULL of a composite reads its fields (a row of NULLs IS NULL), which the copy's text does not
+    if (used.size > 0) {
+        const rows = await catalogRows(catalog, `SELECT oid::text FROM pg_type WHERE typinput::text LIKE '${HELD_ROW_INPUT}%'`);
+        if (rows.length > 0 && nullTested(analysis.tree, new Set(rows.map((r) => Number(r[0]))))) {
+            throw gapRefusal("held-type-compare", "IS [NOT] NULL of a composite column is not supported by Mason: it holds the value's text, and Postgres reads each of its fields", "read x::text, or the source's own columns", text);
+        }
+    }
     // the stored JSON paths the statement's JSON reads may be answered from (src/duck/kept-route.ts)
     const lookup = kept === null ? null : await keptLookup(catalog, analysis.types, kept);
     const keyCounts = await distinctKeys(catalog, analysis.tree);
+    // WHAT EACH COLLATION OF THE ANALYSIS MEANS (src/duck/collation.ts), resolved before the rewrite reads any: a libc
+    // order not read yet is read here, before the plan is made; a collation the source lacks or Mason cannot follow refused
+    let collations;
+    try {
+        collations = await resolver.prepare((sql) => catalogRows(catalog, sql), collationOids(analysis.tree));
+    }
+    catch (e) {
+        throw e instanceof Unrewritable ? unrewritableError(e, text) : e;
+    }
     const analysed = performance.now();
-    const rewritten = (l = lookup) => rewrite(stmt, base, analysis, collation, catalogTypes, sourceRows, timeZone, enums, l, wireBound, keyCounts);
+    const rewritten = (l = lookup) => rewrite({ stmt: stmt, base, analysis, collations, userTypes: catalogTypes, citextHeld, rowTypes: sourceRows, timeZone, dateStyle, enums, kept: l, wireBound, keyCounts });
     try {
-        ({ tree, dropLast, exact, clock, jsonReads, unboundedLateral } = rewritten());
+        ({ tree, withTies, exact, clock, jsonReads, unboundedLateral } = rewritten());
         // the oracle's (`eager`): every read stored before the plan is made, and the plan made again over them
         if (kept?.keepNow != null && lookup !== null && jsonReads.some((r) => kept.ready(r) === null)) {
             await kept.keepNow(jsonReads);
-            ({ tree, dropLast, exact, clock, jsonReads, unboundedLateral } = rewritten());
+            ({ tree, withTies, exact, clock, jsonReads, unboundedLateral } = rewritten());
         }
         // a statement that lists a copy's columns without naming them reads none from a stored column (hideKept): made
         // again with every read its call, the reads still recorded
         if (lookup !== null && exposes(tree))
-            ({ tree, dropLast, exact, clock, jsonReads, unboundedLateral } = rewritten({ at: (relid, attno) => lookup.at(relid, attno), ready: () => null }));
+            ({ tree, withTies, exact, clock, jsonReads, unboundedLateral } = rewritten({ at: (relid, attno) => lookup.at(relid, attno), ready: () => null }));
     }
     catch (e) {
-        // a gap says its id (docs/SQL-SUPPORT.md); Postgres's own refusal is its SQLSTATE, and no gap
-        if (e instanceof Unrewritable) {
-            throw new StatementError(typeof e.gap === "string"
-                ? { code: "0A000", message: e.message, hint: e.hint, detail: `${gapMark(e.gap)} (docs/SQL-SUPPORT.md)` }
-                : { code: e.gap.postgres, message: e.message, hint: e.hint }, text);
-        }
+        if (e instanceof Unrewritable)
+            throw unrewritableError(e, text);
         throw e;
     }
     // a statement that lists a copy's columns without naming them reads its visible ones (src/duck/kept-route.ts); before
@@ -229,7 +542,13 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
     if (kept !== null)
         hideKept(tree, { visible: kept.visible(), searchPath: searchSchemas(searchPath) });
     // a sorted, limited select list is computed over the rows the limit keeps (src/duck/late.ts)
-    tree = selectListAfterLimit(tree, analysis.tree) ?? tree;
+    // (a role's walled read it fetches through itself is one more wall: counted, so `wallStatistics` still walls every one)
+    const lateWall = walls !== null ? { alias: walls.alias, added: 0 } : null;
+    tree = await selectListAfterLimit(tree, analysis.tree, () => joinKeys(catalog, analysis.tree), { volatile: (functions) => volatileFunctions(catalog, functions), wall: lateWall }) ?? tree;
+    // a top N and a min or max under a libc order keyed for their candidates alone (src/duck/libc-bound.ts): after the
+    // late select list, whose inner query holds the sort and the limit; a walled read it reads again is one more wall
+    tree = await libcBounded(tree, analysis.tree, { order: (collate) => resolver.libcOrder(collate),
+        volatile: (functions) => volatileFunctions(catalog, functions), wall: lateWall }) ?? tree;
     // ONLY WHAT THE ORACLE VERIFIED: after the rewrite, so a construct the rewrite refuses by name (with its own way out)
     // says so first
     const keys = [...resolved].sort();
@@ -237,23 +556,26 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
     if (unverified.length > 0) {
         const names = await describeKeys((sql) => catalogRows(catalog, sql), unverified);
         const listed = unverified.map((k) => names.get(k) ?? k);
-        throw new StatementError({ code: "0A000",
-            message: `Mason cannot yet answer this statement on DuckDB exactly as Postgres would: it uses ${listed.slice(0, 5).join("; ")}${listed.length > 5 ? ` and ${listed.length - 5} more` : ""}, which Mason has not verified`,
-            hint: "write the statement without it (docs/SQL-SUPPORT.md lists what is verified), or report it: a statement Mason has not verified is refused rather than answered differently",
-            detail: `${gapMark("unverified")}: ${unverified.join(", ")}` }, text);
+        throw gapRefusal("unverified", `Mason cannot yet answer this statement exactly as Postgres would: it uses ${listed.slice(0, 5).join("; ")}${listed.length > 5 ? ` and ${listed.length - 5} more` : ""}, which Mason has not verified`, "write the statement without it (docs/SQL-SUPPORT.md lists what is verified), or report it: a statement Mason has not verified is refused rather than answered differently", text, `: ${unverified.join(", ")}`);
     }
-    // DuckDB's COLUMNS(t.*) (decidedByKey's star) is an expression only unquoted; the deparser quotes the keyword. No
-    // statement Postgres accepted calls a function "columns" (Postgres has none), and the read-back below checks the text
-    const sql = deparseSync(tree, { pretty: false }).replaceAll('any_value("columns"(', "any_value(COLUMNS(");
+    // only a statement with its filtered reads in place: no other statement's text changes, whatever it names its subqueries
+    if (walls !== null && wallSwitch.statistics)
+        wallStatistics(tree, { alias: walls.alias, reads: walls.reads + (lateWall?.added ?? 0) });
+    // on the finished tree, so no rule above meets it; `masonCalls` below does not count it as a function called
+    giveTurns(tree);
+    if (relations !== undefined && relations.size > 0)
+        swapRelations(tree, relations, searchSchemas(searchPath));
+    const sql = printed(tree, text);
     // THE PRINTED TEXT MUST READ BACK AS THE TREE: a printing gap (the deparser drops or respells something) would run
     // another statement than the one Postgres analysed — refused instead, by name
     const back = (await parseSql(sql)).stmts?.[0]?.stmt;
     const difference = back === undefined ? "nothing" : treeDifference(back, tree);
     if (difference !== null) {
-        throw new StatementError({ code: "0A000", message: "Mason cannot run this statement on DuckDB exactly: its rewritten form does not read back as itself (a Mason gap: report it, with the statement)",
-            detail: `read back: ${difference}; DuckDB SQL: ${sql}` }, text);
+        throw new StatementError({ code: "0A000", message: "Mason cannot run this statement exactly: its rewritten form does not read back as itself (a Mason gap: report it, with the statement)",
+            detail: `read back: ${difference}; the rewritten SQL: ${sql}` }, text);
     }
     const planned = { text: sql, functions: masonCalls(tree), ...(unboundedLateral ? { unboundedLateral: true } : {}) };
-    return { sql: planned, params: analysis.params, fields: fields?.map((f) => ({ ...f, size: pgTyplen(f.oid) })) ?? null, dropLast, ...(exact.length > 0 ? { exact } : {}), resolved: keys,
-        ...(Object.keys(clock).length > 0 ? { clock } : {}), ...(jsonReads.length > 0 ? { jsonReads: jsonReads.map(readKey) } : {}), timing: { analysisMs: analysed - started, rewriteMs: performance.now() - analysed } };
+    return { sql: planned, params: analysis.params, fields: fields?.map((f) => ({ ...f, size: pgTyplen(f.oid) })) ?? null, withTies, ...(exact.length > 0 ? { exact } : {}), resolved: keys, argumentTypes: argumentTypes(analysis.tree), ...(outputs ? { outputs: outputExpressions(analysis.tree) ?? [] } : {}),
+        ...(Object.keys(clock).length > 0 ? { clock } : {}), ...(jsonReads.length > 0 ? { jsonReads: jsonReads.map(readKey) } : {}), ...(reads !== undefined ? { reads } : {}),
+        timing: { analysisMs: analysed - started, rewriteMs: performance.now() - analysed } };
 }