@mason-data/cli 0.6.2 → 0.8.0

package/dist/src/api/postgres/catalog.js

@@ -31,9 +31,13 @@
  * The instance's TimeZone is UTC (the API treats every time as UTC, §4.8); its search_path is Postgres's default, so
  * `public."orders"` and `orders` both resolve, as a tool expects.
  */
+import { randomBytes } from "node:crypto";
 import { Worker } from "node:worker_threads";
 import { fileURLToPath } from "node:url";
 import { runnable } from "../../lib/runnable.js";
+import { NO_ROLE_CATALOG } from "../../duck/grants.js";
+import { renderWhere, tableSql } from "../../duck/row-filter.js";
+import { cstr, frame, splitFrames } from "./protocol.js";
 import { PG_API_TYPE_SPELLING, typeNameParts } from "./schema.js";
 /** Ten minutes: long enough to outlive a tool's startup burst and a person's pause between two queries. */
 const DEFAULT_IDLE_MS = 10 * 60_000;
@@ -46,6 +50,7 @@ export const DEFAULT_STATEMENT_MS = 10_000;
 export const MIN_START_MS = 30_000;
 /** A SQL identifier, double-quoted with an inner quote doubled. */
 const ident = (name) => `"${name.replaceAll('"', '""')}"`;
+const quoteLiteral = (text) => `'${text.replaceAll("'", "''")}'`;
 /** A SQL string literal, single-quoted with an inner quote doubled. */
 const literal = (text) => `'${text.replaceAll("'", "''")}'`;
 /** a table's name in the catalog: its schema (Mason's source schema, else public) and name */
@@ -70,11 +75,11 @@ function isForeignKey(schema, table, join) {
 }
 const joinNote = (join) => `joins ${join.table} on ${join.on[1]} (${join.relationship})`;
 /**
- * THE CATALOG'S DATABASE IS THE ONE A TOOL CONNECTED TO (the deployment): current_database(), pg_database — pgjdbc's
+ * THE CATALOG'S DATABASE IS THE ONE A TOOL CONNECTED TO (src/lib/database-name.ts): current_database(), pg_database — pgjdbc's
  * getCatalogs and every TABLE_CAT, DBeaver's navigator — once said the sidecar's own `postgres`. The current
  * database's row is renamed (ALTER DATABASE refuses the current one; the catalog row does not), after moving aside any
  * OTHER database already holding the name — PGlite's `template0` / `template1` — since pg_database's names are unique
- * (#484's review: a deployment named `template1` failed on pg_database_datname_index and the catalog never started).
+ * (#484's review: a name of `template1` failed on pg_database_datname_index and the catalog never started).
  */
 export function databaseNameSql(name) {
     const q = `'${name.replaceAll("'", "''")}'`;
@@ -163,7 +168,42 @@ export function catalogDdl(schema) {
                 statements.push(`COMMENT ON COLUMN ${qualified(table)}.${ident(c.name)} IS ${literal(parts.join("; "))};`);
         }
     }
-    return [...statements, ...constraints];
+    return [...statements, ...constraints, ...sourceTypeNames(schema)];
+}
+/**
+ * THE NAME OF A TYPE BY THE OID THE WIRE SENDS FOR IT. A column of a type the source made (an enum, a citext) is
+ * sent with the SOURCE's OID (src/duck/plan.ts `userTypes`: what the source's own clients cached), which in this
+ * catalog means another type or none (both count from 16384): `format_type(<that oid>, …)` answered `???`, or another
+ * table's row type. Two readers ask by two OID spaces, and one OID can be in both:
+ *   - `public.mason_wire_type(oid, integer)` names an OID the WIRE sent: a source OID as its type here, every other as
+ *     Postgres does. `mason mcp` names a RowDescription's columns by it; this map is the one place a wire OID is named;
+ *   - the catalog's `format_type(oid, integer)` — the `version()` override's way, search_path listing pg_catalog after
+ *     public — keeps the catalog's meaning for an OID the catalog has (a reader of pg_attribute.atttypid asks by those)
+ *     and names any other by the wire's map. `pg_catalog.format_type`, written out, is Postgres's own.
+ */
+export function sourceTypeNames(schema) {
+    const names = new Map();
+    for (const t of schema.tables) {
+        if (t.rowTypeOid !== undefined)
+            names.set(t.rowTypeOid, qualified(t));
+        for (const c of t.columns) {
+            if (c.typeOid === undefined || c.spelled === undefined)
+                continue;
+            // an array column's own OID is its array type's; an element's array type is the element's spelling with []
+            names.set(c.typeOid, c.spelled);
+            if (c.typeArrayOid !== undefined && !c.spelled.endsWith("[]"))
+                names.set(c.typeArrayOid, `${c.spelled}[]`);
+        }
+    }
+    // with no type the source made, every OID the wire sends is Postgres's: the wire's name is format_type's
+    if (names.size === 0)
+        return [`CREATE OR REPLACE FUNCTION public.mason_wire_type(oid, integer) RETURNS text LANGUAGE sql STABLE AS $$ SELECT pg_catalog.format_type($1, $2) $$;`];
+    const values = [...names].map(([oid, spelled]) => `(${oid}::oid, ${literal(spelled)})`).join(", ");
+    return [`CREATE OR REPLACE FUNCTION public.mason_wire_type(oid, integer) RETURNS text LANGUAGE sql STABLE AS $$
+  SELECT coalesce((SELECT pg_catalog.format_type(pg_catalog.to_regtype(v.spelled), $2) FROM (VALUES ${values}) AS v(source, spelled) WHERE v.source = $1),
+    pg_catalog.format_type($1, $2)) $$;`,
+        `CREATE OR REPLACE FUNCTION public.format_type(oid, integer) RETURNS text LANGUAGE sql STABLE AS $$
+  SELECT CASE WHEN EXISTS (SELECT FROM pg_catalog.pg_type t WHERE t.oid = $1) THEN pg_catalog.format_type($1, $2) ELSE public.mason_wire_type($1, $2) END $$;`];
 }
 /**
  * The source's column defaults, one statement each, run after the schema's DDL by
@@ -199,15 +239,189 @@ export function catalogDefaults(schema) {
  * The source's views, one statement each beside `catalogDefaults`'s: a view reading a table
  * Mason has not copied (or a function the catalog lacks) is said in the log and left out. Each definition qualifies
  * every name outside pg_catalog (src/cli/preflight.ts VIEWS_SQL), so it reads the same tables here; a view over a view
- * is created once the other is (the worker runs these in passes).
+ * is created once the other is (the worker runs these in passes). Every role may SEE every view (`catalogRoles`: the
+ * grant is in the view's own statement, since the view must exist first): reading one is held to the tables it reads.
  */
-export function catalogViews(schema) {
+export function catalogViews(schema, roles = new Map()) {
     return (schema.views ?? []).map((v) => {
         const name = `${ident(v.schema)}.${ident(v.name)}`;
         const comment = v.description ? ` COMMENT ON VIEW ${name} IS ${literal(v.description)};` : "";
-        return `CREATE SCHEMA IF NOT EXISTS ${ident(v.schema)}; CREATE VIEW ${name} AS ${v.definition.replace(/;\s*$/, "")};${comment}`;
+        // every role of the file may see a view; the catalog's role of a user held to nothing sees none (as a login with no
+        // grant on Postgres)
+        const granted = [...roles.keys()].filter((r) => r !== NO_ROLE_CATALOG).map((r) => ` GRANT USAGE ON SCHEMA ${ident(v.schema)} TO ${ident(r)}; GRANT SELECT ON ${name} TO ${ident(r)};`).join("");
+        return `CREATE SCHEMA IF NOT EXISTS ${ident(v.schema)}; CREATE VIEW ${name} AS ${v.definition.replace(/;\s*$/, "")};${comment}${granted}`;
     });
 }
+/**
+ * THE CATALOG'S ROLES MIRROR THE ROLES FILE (GitHub issue 30; src/cli/roles.ts, src/duck/grants.ts): one Postgres role
+ * per Mason role, NOLOGIN, with `GRANT SELECT` on each table the role reads whole and `GRANT SELECT (columns)` on
+ * each it reads in part, and USAGE on every schema the copied tables are in. A session with a role runs its catalog
+ * statements under `SET ROLE` (`asRole`), so `information_schema.tables` / `.columns`, `has_table_privilege` and
+ * `has_column_privilege` answer for the role as Postgres's own do — and `mason mcp`'s `tables` and `describe`, which
+ * ask them, list a role what it may read. `pg_catalog` itself (pg_class, pg_attribute) does not filter by privilege
+ * on Postgres either; it still lists every synced column, said in docs/POSTGRES-API.md. A table the role names that the
+ * schema does not hold yet (added, not copied) is granted at the rebuild its copy brings; a column the copy lacks is
+ * left out (the roles file was held to the copy already, src/cli/roles.ts). Rebuilt with the schema: the server
+ * replaces the catalog when the roles move, as it does when the tables do. **`set_config` is revoked from PUBLIC
+ * while roles exist** (the review of #35): the catalog's one connection is the superuser's with `SET ROLE` on top, so
+ * inside one statement `set_config('role', 'none', false)` was a permitted RESET ROLE and the statement's later
+ * targets ran as the superuser (`information_schema` listed every hidden column's name); revoked, a role's session is
+ * refused it (42501 `permission denied for function set_config`) and the superuser keeps it. A role of `tables: all`
+ * (stage 1b) is resolved to a whole-table grant on every copied table (src/cli/roles.ts resolveRoles), so it is written
+ * here as per-table `GRANT SELECT`s, never `GRANT SELECT ON ALL TABLES IN SCHEMA`: the catalog is rebuilt from the
+ * resolved grants whenever the copied tables move, so a table added later is granted at that rebuild. The file's
+ * `default:` role is one of these roles: a user with no role of its own runs its catalog requests under it; under a
+ * file with no default it runs them as `NO_ROLE_CATALOG` (src/duck/grants.ts catalogRolesOf), a role with no grant.
+ */
+export function catalogRoles(schema, roles) {
+    const out = [];
+    if (roles.size === 0)
+        return out;
+    out.push("REVOKE EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) FROM PUBLIC;");
+    const schemas = [...new Set(schema.tables.map((t) => t.schema ?? "public"))];
+    const byName = new Map(schema.tables.map((t) => [`${t.schema ?? "public"}.${t.name}`, t]));
+    for (const r of roles.values()) {
+        const role = ident(r.name);
+        out.push(`CREATE ROLE ${role} NOLOGIN;`);
+        for (const s of schemas)
+            out.push(`GRANT USAGE ON SCHEMA ${ident(s)} TO ${role};`);
+        for (const g of r.tables.values()) {
+            const t = byName.get(g.table);
+            if (t === undefined)
+                continue;
+            if (g.all) {
+                out.push(`GRANT SELECT ON ${qualified(t)} TO ${role};`);
+                continue;
+            }
+            const columns = g.columns.filter((c) => t.columns.some((x) => x.name === c));
+            if (columns.length > 0)
+                out.push(`GRANT SELECT (${columns.map(ident).join(", ")}) ON ${qualified(t)} TO ${role};`);
+        }
+    }
+    return out;
+}
+/**
+ * THE ROLES' ROW FILTERS AS THE CATALOG'S OWN ROW SECURITY (src/duck/row-filter.ts): every copied table some role filters
+ * gets `ENABLE ROW LEVEL SECURITY` and one permissive SELECT policy per role that reads it — its `where:` with each
+ * attribute read as `current_setting('mason.attr.<name>')::<type>` (a list `string_to_array(…, ',')::<type>`), the type
+ * Postgres inferred for it (`filterTypes`, carried on the roles as `filterTypes`), each request carrying the session's
+ * attributes as settings (`asSession`); a `through:` as `EXISTS (SELECT 1 FROM <parent> …)`, the parent's own policy
+ * applying inside; `true` for a role that reads it unfiltered (row security with no policy hides every row). So
+ * `pg_policies`, `relrowsecurity` and the catalog's own analysis see what Postgres would. The catalog holds no row: the
+ * rows Mason answers are filtered on the copy (src/duck/plan.ts), never here. A filter whose types are not known (its
+ * check failed: the roles' problem refuses every user held to them) gets no policy. Idempotent: each policy dropped first.
+ */
+export function catalogPolicies(schema, roles) {
+    const filtered = new Set([...roles.values()].flatMap((g) => [...(g.filters?.keys() ?? [])]));
+    const out = [];
+    for (const t of schema.tables) {
+        const name = `${t.schema ?? "public"}.${t.name}`;
+        if (!filtered.has(name))
+            continue;
+        out.push(`ALTER TABLE ${qualified(t)} ENABLE ROW LEVEL SECURITY`);
+        for (const g of roles.values()) {
+            if (!g.tables.has(name))
+                continue;
+            const f = g.filters?.get(name);
+            let using;
+            if (f === undefined)
+                using = "true";
+            else if (f.through !== undefined)
+                using = `EXISTS (SELECT 1 FROM ${tableSql(f.through.parent)} AS __mason_parent WHERE __mason_parent.${ident(f.through.parentColumn)} = ${qualified(t)}.${ident(f.through.column)})`;
+            else {
+                const types = g.filterTypes?.get(name);
+                if (types === undefined || f.where === undefined)
+                    continue;
+                const shapes = g.attributes ?? new Map();
+                using = renderWhere(f.where.parts, (a) => {
+                    const read = `current_setting('mason.attr.${a}')`;
+                    return shapes.get(a) === "list" ? `string_to_array(${read}, ',')::${types.get(a) ?? "text[]"}` : `${read}::${types.get(a) ?? "text"}`;
+                });
+            }
+            out.push(`DROP POLICY IF EXISTS ${ident(`mason_${g.name}`)} ON ${qualified(t)}`);
+            out.push(`CREATE POLICY ${ident(`mason_${g.name}`)} ON ${qualified(t)} AS PERMISSIVE FOR SELECT TO ${ident(g.name)} USING (${using})`);
+        }
+    }
+    return out;
+}
+/**
+ * A SESSION'S CATALOG REQUEST AS ITS ROLE: `SET ROLE` before the request's frames and `RESET ROLE` after, in ONE
+ * message — the catalog runs one request at a time (`PgCatalog`'s chain) and a message whole, so no other session's
+ * request runs under this role, and none of this session's runs without it. A Sync is added before the RESET where
+ * the request has none: after an error Postgres ignores every message until a Sync (ignore_till_sync), the RESET
+ * included, and a role left set would be every session's.
+ */
+export function asRole(message, role) {
+    const own = Buffer.from(message.buffer, message.byteOffset, message.byteLength);
+    const synced = splitFrames(own).some((f) => f.type === "S");
+    const set = `SET ROLE ${ident(role)}`;
+    return Buffer.concat([frame("Q", cstr(set)), own, ...(synced ? [] : [frame("S")]), frame("Q", cstr("RESET ROLE"))]);
+}
+/**
+ * The reply to an `asRole` message, less the SET's and the RESET's own replies (each a CommandComplete and a
+ * ReadyForQuery; the Sync added by `asRole` leaves its ReadyForQuery, which every caller drops). `setRoleRefused`: the
+ * SET ROLE's ErrorResponse, when the catalog has no such role (its roles are a rebuild behind the file).
+ */
+export function unwrapRole(frames) {
+    const first = frames[0];
+    if (first === undefined)
+        return { frames: [], setRoleRefused: null };
+    if (first.type === "E")
+        return { frames: [], setRoleRefused: first.frame };
+    // the SET's CommandComplete and ReadyForQuery first; the RESET's CommandComplete and ReadyForQuery last
+    const inner = frames.slice(first.type === "C" && frames[1]?.type === "Z" ? 2 : 0);
+    const n = inner.length;
+    const end = n >= 2 && inner[n - 1].type === "Z" && inner[n - 2].type === "C" ? n - 2 : n;
+    return { frames: inner.slice(0, end), setRoleRefused: null };
+}
+/** the placeholder parameter a session's settings ride in, for one request of its (`asSession`): named anew by each
+ *  process, so no name a tool writes is it — `current_setting('mason.session')` answered the carrier's JSON where
+ *  Postgres (and `SHOW`, and the same beside a copied table) answers 42704 (a review finding of #46). A placeholder
+ *  stays defined once SET, so the name, not a RESET, is what keeps it out of reach. */
+export const SESSION_SETTINGS = `mason_${randomBytes(8).toString("hex")}.session`;
+/**
+ * THE CATALOG'S `current_setting(name[, missing_ok])` IS THE SESSION'S (GitHub issue 40: after `SET TimeZone` it answered
+ * the catalog's UTC, where `SHOW TimeZone` answered the session's): the value the session holds under that name, from
+ * the request's `SESSION_SETTINGS` (`asSession`: every parameter the listener's `sessionSetting` answers), else
+ * Postgres's own (`block_size`; its 42704, or NULL under missing_ok, for a name nobody knows). The `version()`
+ * override's way: search_path lists pg_catalog after public, so a statement reading no table and calling it is
+ * forwarded as ever; one that writes `pg_catalog.current_setting` is spliced to this one (server.ts
+ * userSpliced: Postgres's built-in runs its C function by OID, so its catalog row cannot be replaced). Plain
+ * SQL (plpgsql would load its library, and SHOW ALL list its `plpgsql.*`); STRICT, as Postgres's.
+ */
+export function currentSettingSql() {
+    const held = `nullif(pg_catalog.current_setting('${SESSION_SETTINGS}', true), '')::jsonb ->> lower($1)`;
+    return [
+        `CREATE OR REPLACE FUNCTION public.current_setting(text) RETURNS text LANGUAGE sql STABLE STRICT AS $$ SELECT coalesce(${held}, pg_catalog.current_setting($1)) $$`,
+        `CREATE OR REPLACE FUNCTION public.current_setting(text, boolean) RETURNS text LANGUAGE sql STABLE STRICT AS $$ SELECT coalesce(${held}, pg_catalog.current_setting($1, $2)) $$`,
+    ];
+}
+/**
+ * A SESSION'S CATALOG REQUEST WITH ITS SETTINGS: `SET <SESSION_SETTINGS> = '<its settings as JSON>'` before the request's
+ * frames and `RESET` after, in ONE message, as `asRole` wraps a role (around it, when the session has one) — the catalog
+ * runs one request at a time and a message whole, so no other session's request reads these settings, and none of this
+ * session's runs without them. A Sync is added before the RESET where the request has none (ignore_till_sync: a
+ * setting left set would be every session's). `settings` are the listener's `sessionSetting`, never read here. The
+ * catalog's search_path is pinned in the same SET (`public, pg_catalog`): the overrides of `current_setting` and
+ * `version()` win only while public comes first, and the request must never depend on what the catalog was left with.
+ */
+export function asSession(message, settings) {
+    const own = Buffer.from(message.buffer, message.byteOffset, message.byteLength);
+    const frames = splitFrames(own);
+    const synced = frames.some((f) => f.type === "S") || frames.at(-1)?.type === "Q";
+    const json = JSON.stringify(Object.fromEntries(settings));
+    return Buffer.concat([frame("Q", cstr(`SET search_path = public, pg_catalog; SET ${SESSION_SETTINGS} = ${quoteLiteral(json)}`)), own, ...(synced ? [] : [frame("S")]), frame("Q", cstr(`RESET ${SESSION_SETTINGS}`))]);
+}
+/** The reply to an `asSession` message, less the SETs' and the RESET's own replies (two CommandCompletes and a
+ *  ReadyForQuery first, a CommandComplete and a ReadyForQuery last). A SET the catalog refused is a Mason bug, said. */
+export function unwrapSession(frames) {
+    const ok = frames[0]?.type === "C" && frames[1]?.type === "C" && frames[2]?.type === "Z";
+    const n = frames.length;
+    if (!ok || n < 5 || frames[n - 1].type !== "Z" || frames[n - 2].type !== "C") {
+        throw new Error(`the catalog Postgres did not take the session's settings around a request (${frames.map((f) => f.type).join(", ")}): report it`);
+    }
+    return frames.slice(3, n - 2);
+}
 /**
  * WHAT A LEFT-OUT DEFAULT OR VIEW IS SAID AS, in one line (the test server's log printed every left-out source
  * view's whole definition at each start of the instance — 7 multi-line entries per start, most of a 190 KB log, their
@@ -222,7 +436,7 @@ export function leftOutLine(r) {
         return `catalog: a source default left out (${r.message}): ${r.statement}`;
     const missing = /^relation "([^"]+)" does not exist$/.exec(r.message)?.[1];
     const name = `${unquote(view[1])}.${unquote(view[2])}`;
-    return missing !== undefined ? `catalog: view ${name} left out: it reads ${missing}, which is not copied` : `catalog: view ${name} left out (${r.message})`;
+    return missing !== undefined ? `catalog: view ${name} left out: it reads ${missing}, which is not synced` : `catalog: view ${name} left out (${r.message})`;
 }
 const errorMessage = (e) => (e instanceof Error ? e.message : String(e));
 /** One worker thread holding one PGlite: requests one at a time (PgCatalog's chain), each with a deadline. */
@@ -268,7 +482,8 @@ class CatalogWorker {
         return new Promise((resolve, reject) => {
             const id = this.#next++;
             const timer = setTimeout(() => {
-                this.#die(new Error(`the catalog did not answer ${what} within ${ms / 1000} s: PGlite was discarded (the next statement starts a fresh one; MASON_PG_CATALOG_TIMEOUT_MS raises the deadline)`));
+                // the message reaches the client on the wire and the request log: Mason's words, never the catalog's engine's name
+                this.#die(new Error(`the catalog did not answer ${what} within ${ms / 1000} s, so Mason started it again (the next statement is answered as usual; MASON_PG_CATALOG_TIMEOUT_MS raises the deadline)`));
                 void this.#worker.terminate();
             }, ms);
             timer.unref();
@@ -283,6 +498,7 @@ class CatalogWorker {
 }
 export class PgCatalog {
     #schema;
+    #roles;
     #idleMs;
     #statementMs;
     #log;
@@ -296,8 +512,12 @@ export class PgCatalog {
     #chain = Promise.resolve();
     #idleTimer = null;
     #lastUsed = 0;
+    /** the roles' row security policies (`catalogPolicies`), written once the listener checked the filters: run on the
+     *  instance up now and at every start after, until the schema or the roles are replaced */
+    #policies = [];
     constructor(schema, opts = {}) {
         this.#schema = schema;
+        this.#roles = opts.roles ?? new Map();
         this.#idleMs = opts.idleMs ?? DEFAULT_IDLE_MS;
         this.#statementMs = opts.statementMs ?? DEFAULT_STATEMENT_MS;
         this.#log = opts.log ?? ((line) => console.error(line));
@@ -332,13 +552,36 @@ export class PgCatalog {
     stop() {
         return this.#enqueue(() => this.#close("stopped"));
     }
-    /** Swaps in a new schema (after an apply): the running instance is stopped and the next `run` starts with the new
-     *  DDL. Resolves once the old instance is gone; a caller need not wait for it. */
-    replace(schema) {
+    /** Swaps in a new schema (after an apply), and the roles with it (the roles file moved): the running instance is
+     *  stopped and the next `run` starts with the new DDL. Resolves once the old instance is gone; a caller need not wait for it. */
+    replace(schema, roles = this.#roles) {
         this.#schema = schema;
+        this.#roles = roles;
+        this.#policies = [];
         this.#said.clear();
         return this.#enqueue(() => this.#close("replaced"));
     }
+    /**
+     * THE ROLES' ROW SECURITY POLICIES (`catalogPolicies`), written now (the instance started where needed) and at every
+     * start after, until the next `replace`. A statement the catalog refuses is said, never fatal: the rows are the copy's.
+     */
+    policies(ddl) {
+        return this.#enqueue(async () => {
+            this.#policies = ddl;
+            if (ddl.length === 0)
+                return;
+            const db = await this.#ensureStarted();
+            for (const sql of ddl) {
+                try {
+                    await this.#ask(db, { op: "query", sql }, "a row security policy");
+                }
+                catch (e) {
+                    this.#log(`catalog: a row security policy left out (${errorMessage(e)}): ${sql}`);
+                }
+            }
+            this.#touch();
+        });
+    }
     #enqueue(fn) {
         const result = this.#chain.then(fn);
         // the chain only ORDERS calls: a rejection reaches the caller through `result`, and the next call must still run
@@ -366,11 +609,12 @@ export class PgCatalog {
         if (this.#db !== null && this.#db.alive)
             return this.#db;
         this.#db = null;
-        const schema = this.#schema;
+        const schema = this.#schema, roles = this.#roles;
         const t0 = Date.now();
         const db = new CatalogWorker();
         try {
-            const started = await this.#ask(db, { op: "init", statements: ["SET TimeZone = 'UTC'", ...catalogDdl(schema), ...this.#setup], optional: [...catalogDefaults(schema), ...catalogViews(schema)] }, "its start");
+            const started = await this.#ask(db, { op: "init", statements: ["SET TimeZone = 'UTC'", ...catalogDdl(schema), ...catalogRoles(schema, roles), ...this.#setup],
+                optional: [...catalogDefaults(schema), ...catalogViews(schema, roles), ...this.#policies] }, "its start");
             // each left-out default or view: one line, once per schema (the instance starts again after every idle stop)
             for (const r of started.refused ?? []) {
                 const line = leftOutLine(r);
@@ -388,7 +632,7 @@ export class PgCatalog {
             throw new Error(`PGlite could not start: ${errorMessage(e)}`, { cause: e });
         }
         this.#db = db;
-        this.#log(`catalog: PGlite started in ${Date.now() - t0} ms with ${schema.tables.length} table(s)`);
+        this.#log(`catalog: PGlite started in ${Date.now() - t0} ms with ${schema.tables.length} table(s)${roles.size > 0 ? ` and ${roles.size} role(s)` : ""}`);
         this.#touch();
         return db;
     }

package/dist/src/api/postgres/encode.js

@@ -82,7 +82,9 @@ export function fieldOf(name, type, format) {
 export class EncodeError extends Error {
     /** the SQLSTATE Postgres raises for the same value (a timestamp past its range is 22008), else XX000: Mason's gap */
     code;
-    constructor(message, code = "XX000") { super(message); this.code = code; }
+    /** the registry entry that refused the value (src/duck/gaps.ts): the request log's `gap`; null for any other error */
+    gap;
+    constructor(message, code = "XX000", gap = null) { super(message); this.code = code; this.gap = gap; }
 }
 const refuse = (type, value, why) => {
     throw new EncodeError(`cannot send ${JSON.stringify(value)} as ${type}: ${why}`);

package/dist/src/api/postgres/explain.js

@@ -1,5 +1,6 @@
 import { parseSql, PostgresSyntaxError } from "./parser.js";
 import { Unsupported } from "./translate.js";
+import { gapMark } from "../../duck/gaps.js";
 const bodyOf = (n) => Object.values(n)[0];
 /** Postgres's defGetBoolean: no value is true; true / false / on / off / 1 / 0 */
 function booleanOf(name, arg, at) {
@@ -84,7 +85,7 @@ export function explainOptions(options) {
             }
             case "generic_plan":
                 if (booleanOf(name, b.arg, at))
-                    throw new Unsupported(`EXPLAIN (GENERIC_PLAN) is not supported by Mason (Mason gap explain-option)`, at, "0A000", "bind the parameters' values and EXPLAIN that: DuckDB plans the statement with them");
+                    throw new Unsupported(`EXPLAIN (GENERIC_PLAN) is not supported by Mason (${gapMark("explain-option")})`, at, "0A000", "bind the parameters' values and EXPLAIN that: Mason plans the statement with them", "explain-option");
                 break;
             case "format": {
                 const v = stringOf(name, b.arg, at);
@@ -93,7 +94,7 @@ export function explainOptions(options) {
                     break;
                 }
                 if (v === "xml" || v === "yaml")
-                    throw new Unsupported(`EXPLAIN (FORMAT ${v.toUpperCase()}) is not supported by Mason (Mason gap explain-option)`, at, "0A000", "write FORMAT TEXT or FORMAT JSON");
+                    throw new Unsupported(`EXPLAIN (FORMAT ${v.toUpperCase()}) is not supported by Mason (${gapMark("explain-option")})`, at, "0A000", "write FORMAT TEXT or FORMAT JSON", "explain-option");
                 throw new Unsupported(`unrecognized value for EXPLAIN option "format": "${v}"`, at, "22023");
             }
             default: throw new Unsupported(`unrecognized EXPLAIN option "${name}"`, at, "42601");

package/dist/src/api/postgres/protocol.js

@@ -75,12 +75,10 @@ class BodyReader {
     rest() { return Buffer.from(this.#buf.subarray(this.#off)); }
     get done() { return this.#off >= this.#buf.length; }
 }
-/** The next frame of `buf` at `off`: its length (type byte excluded, length field included) when the frame is whole,
- *  or null for "need more bytes". `headerLength` is 5 with a type byte, 4 in the startup phase. */
-/** The length field of the frame at the start of `buf` (type byte excluded, the field itself included), checked
- *  against `limit`; `headerLength` is 5 with a type byte, 4 in the startup phase. */
-function frameLength(buf, headerLength, limit) {
-    const len = buf.readInt32BE(headerLength - 4);
+/** The length field of the frame at `off` in `buf` (type byte excluded, the field itself included), checked against
+ *  `limit`; `headerLength` is 5 with a type byte, 4 in the startup phase. */
+function frameLength(buf, off, headerLength, limit) {
+    const len = buf.readInt32BE(off + headerLength - 4);
     if (len < 4)
         throw new ProtocolError(`malformed message: a length of ${len} (the length counts itself, so it is at least 4)`);
     if (len > limit)
@@ -89,8 +87,10 @@ function frameLength(buf, headerLength, limit) {
 }
 /** Decodes what a frontend sends. Push every chunk the socket delivers; call `next()` until it returns null. */
 export class FrontendDecoder {
-    /** the bytes of the frame being assembled, once its header has arrived */
+    /** the chunks joined so far: consumed bytes before `#off`, then the frames not yet read, the last perhaps partial */
     #buf = Buffer.alloc(0);
+    /** where the next frame starts in `#buf`: a frame consumed moves this, never the bytes behind it (BackendDecoder) */
+    #off = 0;
     /** chunks received since, concatenated ONCE, when the frame they complete is whole (never per chunk: a frame in
      *  64 KiB pieces re-copied everything received so far on every piece — review of #480) */
     #pending = [];
@@ -106,13 +106,16 @@ export class FrontendDecoder {
         this.#pending.push(Buffer.from(chunk));
         this.#pendingLength += chunk.length;
     }
-    /** Makes `#buf` hold at least `need` bytes when that many have arrived, moving the pending chunks in with one copy. */
+    /** Makes `#buf` hold at least `need` bytes past `#off` when that many have arrived, moving the pending chunks in with
+     *  one copy, which leaves the consumed bytes behind. */
     #assemble(need) {
-        if (this.#buf.length >= need)
+        const held = this.#buf.length - this.#off;
+        if (held >= need)
             return true;
-        if (this.#buf.length + this.#pendingLength < need)
+        if (held + this.#pendingLength < need)
             return false;
-        this.#buf = Buffer.concat([this.#buf, ...this.#pending]);
+        this.#buf = Buffer.concat([this.#buf.subarray(this.#off), ...this.#pending]);
+        this.#off = 0;
         this.#pending = [];
         this.#pendingLength = 0;
         return true;
@@ -126,12 +129,18 @@ export class FrontendDecoder {
     #whole(headerLength) {
         if (!this.#assemble(headerLength))
             return null;
-        const len = frameLength(this.#buf, headerLength, this.#startup ? Math.min(this.maxLength, MAX_STARTUP_LENGTH) : this.maxLength);
+        const len = frameLength(this.#buf, this.#off, headerLength, this.#startup ? Math.min(this.maxLength, MAX_STARTUP_LENGTH) : this.maxLength);
         return this.#assemble(headerLength - 4 + len) ? len : null;
     }
+    /** The frame's body, a view of `#buf` that lives as long as its BodyReader: every value a reader returns is a copy. */
     #take(headerLength, len) {
-        const body = this.#buf.subarray(headerLength, headerLength - 4 + len);
-        this.#buf = Buffer.from(this.#buf.subarray(headerLength - 4 + len));
+        const body = this.#buf.subarray(this.#off + headerLength, this.#off + headerLength - 4 + len);
+        this.#off += headerLength - 4 + len;
+        // wholly consumed, it is let go: a connection idle on a frame boundary holds no bytes of what it last sent
+        if (this.#off === this.#buf.length) {
+            this.#buf = Buffer.alloc(0);
+            this.#off = 0;
+        }
         return body;
     }
     #nextStartup() {
@@ -165,7 +174,7 @@ export class FrontendDecoder {
         const len = this.#whole(5);
         if (len === null)
             return null;
-        const code = String.fromCharCode(this.#buf[0]);
+        const code = String.fromCharCode(this.#buf[this.#off]);
         const r = new BodyReader(this.#take(5, len), `'${code}'`);
         switch (code) {
             case "p": return { type: "Password", password: r.cstring() };
@@ -353,22 +362,29 @@ function decodeBackendBody(code, body) {
 /** Decodes what a backend sends, chunk by chunk, as FrontendDecoder does for the other direction. */
 export class BackendDecoder {
     #buf = Buffer.alloc(0);
+    /** WHERE THE NEXT FRAME STARTS IN `#buf`: `next()` moves this and `push` drops what is behind it. Copying the rest off
+     *  after every message was quadratic in a reply: a 30 MB reply of 150,000 rows took about six minutes on CI (issue
+     *  77). A new buffer is always allocated, never written over, and no decoded message keeps a view of one (BodyReader
+     *  copies). */
+    #off = 0;
     push(chunk) {
-        this.#buf = this.#buf.length === 0 ? Buffer.from(chunk) : Buffer.concat([this.#buf, chunk]);
+        const rest = this.#buf.subarray(this.#off);
+        this.#buf = rest.length === 0 ? Buffer.from(chunk) : Buffer.concat([rest, chunk]);
+        this.#off = 0;
     }
     next() {
-        if (this.#buf.length < 5)
+        if (this.pending < 5)
             return null;
-        const len = frameLength(this.#buf, 5, MAX_MESSAGE_LENGTH);
-        if (this.#buf.length < 1 + len)
+        const len = frameLength(this.#buf, this.#off, 5, MAX_MESSAGE_LENGTH);
+        if (this.pending < 1 + len)
             return null;
-        const code = String.fromCharCode(this.#buf[0]);
-        const body = this.#buf.subarray(5, 1 + len);
-        this.#buf = Buffer.from(this.#buf.subarray(1 + len));
+        const code = String.fromCharCode(this.#buf[this.#off]);
+        const body = this.#buf.subarray(this.#off + 5, this.#off + 1 + len);
+        this.#off += 1 + len;
         return decodeBackendBody(code, body);
     }
     /** bytes pushed and not yet consumed by a whole message */
-    get pending() { return this.#buf.length; }
+    get pending() { return this.#buf.length - this.#off; }
 }
 /** Every message of a buffer that holds whole messages only (a `execProtocolRaw` reply, a builder's output). A
  *  trailing partial message throws: the caller promised a complete reply. */