@mason-data/cli 0.6.2 → 0.8.0

package/dist/src/duck/plan.js

@@ -6,16 +6,18 @@
  * per text until the schema changes.
  */
 import { deparseSync, QuoteUtils } from "pgsql-deparser";
-import { StatementError } from "../api/postgres/server.js";
+import { gapRefusal, StatementError } from "../api/postgres/server.js";
 import { parseSql } from "../api/postgres/parser.js";
 import { analyze, catalogRows, readParameters } from "./analyze.js";
 import { describeKeys, nameUserKeys, resolvedKeys } from "./resolved.js";
-import { gapMark } from "./gaps.js";
 import { expandViews, MAX_VIEW_DEPTH, viewRefused } from "./views.js";
 import { selectListAfterLimit } from "./late.js";
 import { pgTyplen } from "./types.js";
 import { datumText, masonCalls, rewrite, treeDifference, Unrewritable } from "./rewrite.js";
 import { exposes, hideKept, readKey, searchSchemas } from "./kept-route.js";
+import { distinctCountTables, distinctKeyCounts, uniqueColumnsOf, uniqueColumnsSql } from "./distinct-key.js";
+import { enforceGrants } from "./grants.js";
+import { attributeRefused, expandRowFilters, wallStatistics, wallSwitch } from "./row-filter.js";
 const JSON_OID = 114, JSONB_OID = 3802;
 /** what the rewrite asks of the stored JSON paths (src/duck/kept-route.ts KeptLookup): the copied table and column of
  *  each json or jsonb column the statement reads from a table (Typed.relation), by the catalog's OIDs; null when it
@@ -33,6 +35,15 @@ async function keptLookup(catalog, types, kept) {
     const at = new Map(rows.map(([relid, attno, schema, table, column]) => [`${relid}.${attno}`, { source: `${schema}.${table}`, column: column }]));
     return { at: (relid, attno) => at.get(`${relid}.${attno}`) ?? null, ready: (read) => kept.ready(read) };
 }
+/** the count(DISTINCT x) of the statement that are count(x) (src/duck/distinct-key.ts), by their offsets into the
+ *  analysed text: x unique by the CATALOG's own pg_index — the source's keys as the catalog declares them — read only
+ *  for a statement that counts DISTINCT values at all */
+async function distinctKeys(catalog, tree) {
+    const relids = distinctCountTables(tree);
+    if (relids.length === 0)
+        return new Set();
+    return distinctKeyCounts(tree, uniqueColumnsOf(await catalogRows(catalog, uniqueColumnsSql(relids))));
+}
 /**
  * DuckDB's keywords that Postgres's are not: a column named `at` (DuckDB's AT TIME ZONE) or `qualify` is an identifier
  * the deparser writes bare, as Postgres reads it, and DuckDB refuses as a syntax error. The deparser's one quoting
@@ -96,11 +107,11 @@ export function missingRelationHint(relation, copies) {
     const matches = copies.filter((c) => c.source === relation || (!relation.includes(".") && c.source.endsWith(`.${relation}`)))
         .sort((a, b) => Number(b.source.startsWith("public.")) - Number(a.source.startsWith("public.")));
     const c = matches[0];
-    const served = "it is served once its copy commits; mason wait waits for it (mason status shows how far it is)";
+    const served = "it is served once its first sync commits; mason wait waits for it (mason status shows how far it is)";
     if (c === undefined)
-        return `a table is in Mason once it is added: mason add ${relation} copies it (mason tables lists the database's)`;
+        return `a table is in Mason once it is added: mason add ${relation} syncs it (mason tables lists the database's)`;
     switch (c.phase) {
-        case "waiting": return `${c.source} is added and waits its turn to be copied: ${served}`;
+        case "waiting": return `${c.source} is added and waits its turn to be synced: ${served}`;
         case "copying": {
             // the share is of the heap's PAGES read (`CopyProgressView.share`: exact, so it never moves backwards), the count
             // the rows the committed chunks and the ones reading have in; a copy can outrun the source's estimate (Postgres's
@@ -113,13 +124,55 @@ export function missingRelationHint(relation, copies) {
                 : of === null || of <= 0 ? `${share}about ${so} rows so far`
                     : so > of ? `${share}${so} rows so far, past the ~${of} estimated`
                         : `${share}about ${so} of ~${of} rows so far${p === undefined ? `, ${Math.min(99, Math.floor((so / of) * 100))}%` : ""}`;
-            return `${c.source} is added and being copied (${how}): ${served}`;
+            return `${c.source} is added and being synced (${how}): ${served}`;
         }
-        case "failed": return `${c.source} is added, but its copy failed: ${c.error ?? "no reason recorded"}; mason logs says more`;
-        case "copied": return `${c.source} was just copied and joins the catalog as it is rebuilt: run the statement again in a moment (if it stays missing, mason logs says why)`;
+        case "failed": return `${c.source} is added, but its first sync failed: ${c.error ?? "no reason recorded"}; mason logs says more`;
+        case "copied": return `${c.source} was just synced and joins the catalog as it is rebuilt: run the statement again in a moment (if it stays missing, mason logs says why)`;
     }
 }
-export async function planDuck(catalog, text, stmt, base, paramOids, searchPath, timeZone, collation, verified, userTypes = new Map(), copies = () => [], depth = 0, kept = null) {
+/**
+ * THE TREE AS TEXT, DuckDB's `COLUMNS(…)` (decidedByKey's star, the statistics wall) unquoted: it is an expression only so,
+ * and the deparser quotes the keyword. Marked in the tree, never found in the printed text, where a literal or a name may
+ * hold the same characters: each call printed under a name the text holds nowhere else (the count says so), then that name
+ * replaced. No statement Postgres accepted calls a function "columns" (Postgres has none), and the read-back checks the text.
+ */
+function printed(tree, statement) {
+    const calls = [];
+    const collect = (v) => {
+        if (Array.isArray(v)) {
+            v.forEach(collect);
+            return;
+        }
+        if (typeof v !== "object" || v === null)
+            return;
+        const name = v.FuncCall?.funcname;
+        if (name?.length === 1 && name[0]?.String?.sval === "columns")
+            calls.push(name[0]);
+        Object.values(v).forEach(collect);
+    };
+    collect(tree);
+    if (calls.length === 0)
+        return deparseSync(tree, { pretty: false });
+    // a text holds a mark at most once per few characters; a count that never agrees is a call the deparser does not print
+    const bound = deparseSync(tree, { pretty: false }).length;
+    for (let i = 0; i <= bound; i++) {
+        const mark = `__mason_columns_${i}`;
+        for (const c of calls)
+            c.String.sval = mark;
+        const text = deparseSync(tree, { pretty: false });
+        for (const c of calls)
+            c.String.sval = "columns";
+        if (text.split(`${mark}(`).length - 1 === calls.length)
+            return text.replaceAll(`${mark}(`, "COLUMNS(");
+    }
+    throw new StatementError({ code: "0A000", message: "Mason cannot print this statement for the engine (a Mason gap: report it, with the statement)" }, statement);
+}
+/** A statement's plan. `wireBound`: its rows are bounded by a row count an Execute sets on the wire, which its SQL
+ *  does not say (src/duck/rewrite.ts markBounds; the listener asks for it where PlannedSql.unboundedLateral says so).
+ *  `grants`: the session's role (src/duck/grants.ts), held after the analysis; null for a session with no role where
+ *  roles exist (the plan still names its reads, for an Execute under a role given later); undefined with no roles at all.
+ *  `walls`: `text` is the statement with its filtered reads in place (expandRowFilters), each of which the wall ends */
+export async function planDuck(catalog, text, stmt, base, paramOids, searchPath, timeZone, collation, verified, userTypes = new Map(), copies = () => [], depth = 0, kept = null, wireBound = false, grants = undefined, walls = null) {
     const started = performance.now();
     const analysis = await analyze(catalog, text, paramOids, pathSql(searchPath), timeZone);
     if ("refused" in analysis) {
@@ -138,13 +191,51 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
         if (depth >= MAX_VIEW_DEPTH)
             throw viewRefused(`Mason cannot yet answer views read through more than ${MAX_VIEW_DEPTH} other views`, "read the tables the views read");
         try {
-            return await planDuck(catalog, expanded.text, expanded.stmt, 0, analysis.params, searchPath, timeZone, collation, verified, userTypes, copies, depth + 1, kept);
+            return await planDuck(catalog, expanded.text, expanded.stmt, 0, analysis.params, searchPath, timeZone, collation, verified, userTypes, copies, depth + 1, kept, wireBound, grants, walls);
         }
         catch (e) {
             if (!(e instanceof StatementError))
                 throw e;
             const { position: _p, ...fields } = e.fields;
-            throw new StatementError(fields, text);
+            throw new StatementError(fields, text, e.gap);
+        }
+    }
+    // THE ROLE (src/duck/grants.ts): the tables and columns Postgres's analysis says the statement reads, held to the
+    // session's role — after Postgres's own refusal and the views' expansion (the check sees the tables a view reads),
+    // before the rewrite (a role's refusal before a Mason gap's). The reads stay on the plan: an Execute under a role
+    // given or taken since the Parse is held to them again (src/api/postgres/server.ts)
+    const reads = grants === undefined ? undefined : await enforceGrants(catalog, analysis.tree, grants);
+    // THE ROLE'S ROWS (src/duck/row-filter.ts): every read of a table the role filters becomes its filtered read, and that
+    // statement is planned from the start — analysed by Postgres, rewritten and verified as any other (one grammar), held
+    // to nothing more (the grants were held above, on the statement as written: a filter may read a column the role
+    // may not, as a policy may on Postgres). Its errors are the expanded text's, so they carry no position into this one
+    if (grants?.rows !== undefined && grants.rows.size > 0) {
+        const filtered = await expandRowFilters(catalog, stmt, analysis, pathSql(searchPath), grants.rows);
+        if (filtered !== null) {
+            let inner;
+            try {
+                inner = await planDuck(catalog, filtered.text, filtered.stmt, 0, analysis.params, searchPath, timeZone, collation, verified, userTypes, copies, depth + 1, kept, wireBound, undefined, filtered.walls);
+            }
+            catch (e) {
+                if (!(e instanceof StatementError))
+                    throw e;
+                // a data exception may be a bound value's, not the statement's: `--set tenant=abc` on an integer
+                const bad = e.fields.code.startsWith("22") && e.gap === null ? await attributeRefused(catalog, grants.rows, filtered.tables) : null;
+                if (bad !== null) {
+                    const user = JSON.stringify(grants.rowsUser ?? "");
+                    throw new StatementError({ code: "42501", message: `permission denied: the user ${user} has a value for the attribute ${bad.name} that is not of type ${bad.type}, which the role ${grants.name} filters rows by`,
+                        detail: `${bad.at} reads :${bad.name} as ${bad.type} (${bad.where}): ${bad.problem}`,
+                        hint: `mason user remove ${grants.rowsUser ?? "<user>"} && mason user add ${grants.rowsUser ?? "<user>"} --role ${grants.name} --set ${bad.name}=<a value of type ${bad.type}>` }, text);
+                }
+                // the statement as written passed this check above, so what fails it is the table being a subquery (carryKeyDependency)
+                if (e.fields.code === "42803" || /could not identify an equality operator/.test(e.fields.message)) {
+                    throw gapRefusal("roles", `Mason cannot yet answer a GROUP BY of a key of a table its role filters here: ${e.fields.message}`, "name each column it reads in the GROUP BY, or read it in an aggregate", text);
+                }
+                const { position: _p, ...fields } = e.fields;
+                throw new StatementError(fields, text, e.gap);
+            }
+            const rowFilters = filtered.tables.map((t) => ({ table: t, words: `filtered by the role ${grants.name}: ${grants.rows.get(t).words}` }));
+            return { ...inner, ...(reads !== undefined ? { reads } : {}), rowFilters };
         }
     }
     // the catalog's own OIDs for the types an extension made (citext): the rewrite knows a few by name, and the wire sends
@@ -183,31 +274,31 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
                 f.oid = source;
         }
     }
-    let tree, dropLast, exact, clock, jsonReads;
+    let tree, dropLast, exact, clock, jsonReads, unboundedLateral;
     // a table's row the source has otherwise (copied in part: no source OID for it) is not the source's row
     const sourceRows = new Map([...rowTypes].filter(([oid]) => userTypes.has(catalogTypes.get(oid) ?? "")));
     // the stored JSON paths the statement's JSON reads may be answered from (src/duck/kept-route.ts)
     const lookup = kept === null ? null : await keptLookup(catalog, analysis.types, kept);
+    const keyCounts = await distinctKeys(catalog, analysis.tree);
     const analysed = performance.now();
-    const rewritten = (l = lookup) => rewrite(stmt, base, analysis, collation, catalogTypes, sourceRows, timeZone, enums, l);
+    const rewritten = (l = lookup) => rewrite(stmt, base, analysis, collation, catalogTypes, sourceRows, timeZone, enums, l, wireBound, keyCounts);
     try {
-        ({ tree, dropLast, exact, clock, jsonReads } = rewritten());
+        ({ tree, dropLast, exact, clock, jsonReads, unboundedLateral } = rewritten());
         // the oracle's (`eager`): every read stored before the plan is made, and the plan made again over them
         if (kept?.keepNow != null && lookup !== null && jsonReads.some((r) => kept.ready(r) === null)) {
             await kept.keepNow(jsonReads);
-            ({ tree, dropLast, exact, clock, jsonReads } = rewritten());
+            ({ tree, dropLast, exact, clock, jsonReads, unboundedLateral } = rewritten());
         }
         // a statement that lists a copy's columns without naming them reads none from a stored column (hideKept): made
         // again with every read its call, the reads still recorded
         if (lookup !== null && exposes(tree))
-            ({ tree, dropLast, exact, clock, jsonReads } = rewritten({ at: (relid, attno) => lookup.at(relid, attno), ready: () => null }));
+            ({ tree, dropLast, exact, clock, jsonReads, unboundedLateral } = rewritten({ at: (relid, attno) => lookup.at(relid, attno), ready: () => null }));
     }
     catch (e) {
         // a gap says its id (docs/SQL-SUPPORT.md); Postgres's own refusal is its SQLSTATE, and no gap
         if (e instanceof Unrewritable) {
-            throw new StatementError(typeof e.gap === "string"
-                ? { code: "0A000", message: e.message, hint: e.hint, detail: `${gapMark(e.gap)} (docs/SQL-SUPPORT.md)` }
-                : { code: e.gap.postgres, message: e.message, hint: e.hint }, text);
+            throw typeof e.gap === "string" ? gapRefusal(e.gap, e.message, e.hint, text)
+                : new StatementError({ code: e.gap.postgres, message: e.message, hint: e.hint }, text);
         }
         throw e;
     }
@@ -224,23 +315,22 @@ export async function planDuck(catalog, text, stmt, base, paramOids, searchPath,
     if (unverified.length > 0) {
         const names = await describeKeys((sql) => catalogRows(catalog, sql), unverified);
         const listed = unverified.map((k) => names.get(k) ?? k);
-        throw new StatementError({ code: "0A000",
-            message: `Mason cannot yet answer this statement on DuckDB exactly as Postgres would: it uses ${listed.slice(0, 5).join("; ")}${listed.length > 5 ? ` and ${listed.length - 5} more` : ""}, which Mason has not verified`,
-            hint: "write the statement without it (docs/SQL-SUPPORT.md lists what is verified), or report it: a statement Mason has not verified is refused rather than answered differently",
-            detail: `${gapMark("unverified")}: ${unverified.join(", ")}` }, text);
+        throw gapRefusal("unverified", `Mason cannot yet answer this statement exactly as Postgres would: it uses ${listed.slice(0, 5).join("; ")}${listed.length > 5 ? ` and ${listed.length - 5} more` : ""}, which Mason has not verified`, "write the statement without it (docs/SQL-SUPPORT.md lists what is verified), or report it: a statement Mason has not verified is refused rather than answered differently", text, `: ${unverified.join(", ")}`);
     }
-    // DuckDB's COLUMNS(t.*) (decidedByKey's star) is an expression only unquoted; the deparser quotes the keyword. No
-    // statement Postgres accepted calls a function "columns" (Postgres has none), and the read-back below checks the text
-    const sql = deparseSync(tree, { pretty: false }).replaceAll('any_value("columns"(', "any_value(COLUMNS(");
+    // only a statement with its filtered reads in place: no other statement's text changes, whatever it names its subqueries
+    if (walls !== null && wallSwitch.statistics)
+        wallStatistics(tree, walls);
+    const sql = printed(tree, text);
     // THE PRINTED TEXT MUST READ BACK AS THE TREE: a printing gap (the deparser drops or respells something) would run
     // another statement than the one Postgres analysed — refused instead, by name
     const back = (await parseSql(sql)).stmts?.[0]?.stmt;
     const difference = back === undefined ? "nothing" : treeDifference(back, tree);
     if (difference !== null) {
-        throw new StatementError({ code: "0A000", message: "Mason cannot run this statement on DuckDB exactly: its rewritten form does not read back as itself (a Mason gap: report it, with the statement)",
-            detail: `read back: ${difference}; DuckDB SQL: ${sql}` }, text);
+        throw new StatementError({ code: "0A000", message: "Mason cannot run this statement exactly: its rewritten form does not read back as itself (a Mason gap: report it, with the statement)",
+            detail: `read back: ${difference}; the rewritten SQL: ${sql}` }, text);
     }
-    const planned = { text: sql, functions: masonCalls(tree) };
+    const planned = { text: sql, functions: masonCalls(tree), ...(unboundedLateral ? { unboundedLateral: true } : {}) };
     return { sql: planned, params: analysis.params, fields: fields?.map((f) => ({ ...f, size: pgTyplen(f.oid) })) ?? null, dropLast, ...(exact.length > 0 ? { exact } : {}), resolved: keys,
-        ...(Object.keys(clock).length > 0 ? { clock } : {}), ...(jsonReads.length > 0 ? { jsonReads: jsonReads.map(readKey) } : {}), timing: { analysisMs: analysed - started, rewriteMs: performance.now() - analysed } };
+        ...(Object.keys(clock).length > 0 ? { clock } : {}), ...(jsonReads.length > 0 ? { jsonReads: jsonReads.map(readKey) } : {}), ...(reads !== undefined ? { reads } : {}),
+        timing: { analysisMs: analysed - started, rewriteMs: performance.now() - analysed } };
 }

package/dist/src/duck/publication.js

@@ -1,8 +1,8 @@
 const requiredOf = (t) => t.columns.filter((c) => !c.generated).map((c) => c.name);
 /** what follows every refusal: Mason waits for the owner, and when it looks again */
 const NEVER_ALTERED = "Mason reads a publication it is given as it is, never altering it: once its owner has run that, the next mason start " +
-    "(or mason add) checks it again. Or mason start --replication without --publication, where Mason makes its own (not on a read " +
-    "replica), or mason start --poll, which needs no publication.";
+    "(or mason add) checks it again. Or give the table's entry sync: poll <interval> in the sources file, which needs no publication; " +
+    "or write publication: null in it, where Mason makes its own, carrying the replicated tables alone (not on a read replica).";
 /**
  * The verdict over a described publication (pure: `described` is what `readNamedPublication` read). `identities` are
  * the tables' `relreplident`: a table whose identity is not its key is expected WHOLE (a column list missing an
@@ -56,13 +56,13 @@ function checkPublication(required, described, whole) {
             continue;
         if (d.published === null) {
             problems.push(`${table} is not published`);
-            missingLines.push(`  missing: ${table} (read by Mason's copy)`);
+            missingLines.push(`  missing: ${table} (read by Mason's sync)`);
             continue;
         }
         const lacking = inCatalogOrder(d, cols).filter((c) => !d.published.includes(c));
         if (lacking.length) {
             problems.push(`${table} columns ${lacking.join(", ")} are not published`);
-            missingLines.push(`  missing: ${table} columns ${lacking.join(", ")} (read by Mason's copy)`);
+            missingLines.push(`  missing: ${table} columns ${lacking.join(", ")} (read by Mason's sync)`);
         }
         if (d.rowFilter !== null) {
             problems.push(`${table} is published with a row filter`);
@@ -103,7 +103,7 @@ function checkPublication(required, described, whole) {
             (found.size ? tableLines(found, columnLists).join(",\n") : "    (no tables)")]
         : [`Found: no publication named ${pub.name}. Publications on this database: ${pub.others.length ? pub.others.join(", ") : "none"}.`];
     return [
-        `publication ${pub.name} does not carry what Mason's copy of the table holds.`,
+        `publication ${pub.name} does not carry what Mason's synced table holds.`,
         "",
         ...(missingLines.length ? [...missingLines, ""] : []),
         ...(publishes.length && pub.exists ? [`  it publishes ${pub.publish.join(", ") || "nothing"}; the change reader needs insert, update and delete`, ""] : []),
@@ -185,7 +185,7 @@ export function describedOf(answer, publication) {
 }
 /** The named publication and the copied tables as the live catalog describes them: read-only, two statements.
  *  (`_project`, the project's name, is unused: the tables are found by their own names.) */
-export async function readNamedPublication(pg, publication, tables, _project) {
+export async function readNamedPublication(pg, publication, tables) {
     const one = async (sql) => {
         const v = (await pg.query(sql)).rows[0]?.[0];
         if (typeof v !== "string")