@mason-data/cli 0.6.2 → 0.8.0

package/dist/src/duck/types.js

@@ -13,6 +13,7 @@
 import { Buffer } from "node:buffer";
 import { DuckDBTypeId, DuckDBDateValue, DuckDBTimestampValue, DuckDBTimestampTZValue, DuckDBDecimalValue, DuckDBUUIDValue, DuckDBListValue, DuckDBArrayValue, DuckDBIntervalValue, DuckDBBlobValue, DuckDBTimeValue, DuckDBListType, DuckDBArrayType, DuckDBTimestampSecondsValue, DuckDBTimestampMillisecondsValue, DuckDBTimestampNanosecondsValue, } from "@duckdb/node-api";
 import { jsonbText } from "./jsonb.js";
+import { gapMark } from "./gaps.js";
 import { EncodeError, encodeValue, OID, TIMESTAMP } from "../api/postgres/encode.js";
 const MICROS_PER_DAY = 86400000000n;
 const DATE_INFINITY = 2_147_483_647, DATE_NEG_INFINITY = -2_147_483_647;
@@ -124,7 +125,7 @@ function timestampColumn(name, tz) {
                 }
                 // a zone's offset before year 1 is its local mean time, which Mason does not compute
                 if (tz && s.timeZone.toUpperCase() !== "UTC")
-                    throw new EncodeError(`Mason writes a timestamptz before year 1 only in UTC, not in ${s.timeZone} (Mason gap bc-timestamptz-zone)`, "0A000");
+                    throw new EncodeError(`Mason writes a timestamptz before year 1 only in UTC, not in ${s.timeZone} (${gapMark("bc-timestamptz-zone")})`, "0A000", "bc-timestamptz-zone");
                 return Buffer.from(bcTimestampText(m, tz), "ascii");
             }
             return base.encode(v, format, s);
@@ -267,7 +268,7 @@ export function wireColumn(name, t) {
         case DuckDBTypeId.UHUGEINT:
         case DuckDBTypeId.UBIGINT:
         case DuckDBTypeId.BIGNUM:
-            return viaApi(name, "numeric", bigintText, `DuckDB's ${t.toString()} (a sum of integers is one), sent as numeric: Postgres says bigint for a sum of int4 and numeric for a sum of int8`);
+            return viaApi(name, "numeric", bigintText, `the engine's ${t.toString()} (a sum of integers is one), sent as numeric: Postgres says bigint for a sum of int4 and numeric for a sum of int8`);
         case DuckDBTypeId.FLOAT: return viaApi(name, "float4", (v) => v);
         case DuckDBTypeId.DOUBLE: return viaApi(name, "float8", (v) => v);
         case DuckDBTypeId.DECIMAL: return viaApi(name, "numeric", (v) => v.toString());
@@ -303,7 +304,7 @@ export function wireColumn(name, t) {
         }
         default: break;
     }
-    return custom(name, 25, -1, (v) => String(v), null, `DuckDB's ${t.toString()} has no Postgres type here: sent as its text`);
+    return custom(name, 25, -1, (v) => String(v), null, `the engine's ${t.toString()} has no Postgres type here: sent as its text`);
 }
 // ---------------------------------------------------------------------------------------------------------------------
 // A COLUMN OF POSTGRES'S TYPE (docs/DESIGN.md "Postgres decides, DuckDB computes"): the catalog Postgres's
@@ -399,9 +400,9 @@ export function pgTypedColumn(name, oid, t, typmod = -1) {
     if (oid === 1700 && DUCK_FLOAT.has(t.typeId)) {
         // a double that overflowed is Postgres's numeric overflow (its exact arithmetic raises it)
         return { ...retyped(viaApi(name, "numeric", (v) => { if (v === Infinity || v === -Infinity)
-                throw new EncodeError("value overflows numeric format", "22003"); return floatAsNumeric(v); })), divergence: `Postgres computes this numeric exactly; DuckDB computed it as a double (${t.toString()}), sent with 15 significant digits` };
+                throw new EncodeError("value overflows numeric format", "22003"); return floatAsNumeric(v); })), divergence: `Postgres computes this numeric exactly; the engine computed it as a double (${t.toString()}), sent with 15 significant digits` };
     }
-    return { ...own, divergence: `Postgres says type OID ${oid}; DuckDB computed ${t.toString()}, sent as its own type` };
+    return { ...own, divergence: `Postgres says type OID ${oid}; the engine computed ${t.toString()}, sent as its own type` };
 }
 /** How a cell of an exact numeric (src/duck/rewrite.ts ExactColumn) reads as a decimal text, by DuckDB's type: an exact
  *  number's digits, a float as Postgres's cast to numeric writes it — or null where the cell needs digits DuckDB did not

package/dist/src/duck/usage.js

@@ -0,0 +1,324 @@
+import { MASON_SCHEMA } from "./store.js";
+import { History, Streak } from "./history.js";
+import { outcomeOf } from "../api/request-log.js";
+export const USAGE_TABLE = `${MASON_SCHEMA}.usage`;
+export const SIZES_TABLE = `${MASON_SCHEMA}.copy_sizes`;
+/**
+ * HOW OFTEN A SAMPLE IS TAKEN: a minute. A chart of a day is 1,440 points and of the retention's 7 days 10,080 (a few
+ * hundred KB of rows); a statement's p95 over a minute of a BI tool's traffic is a number, over 5 s mostly one statement;
+ * and a sample is a read of the state the commands already ask every few seconds. No setting: nothing an operator
+ * would tune it for has been met.
+ */
+export const SAMPLE_SECONDS = 60;
+/** how often each copy's own bytes are measured (`copySizes`): its cost grows with the copy, see above */
+export const SIZE_EVERY_MS = 10 * 60_000;
+const COLUMNS = ["sampled_at", "seconds", "copy_bytes", "copy_rows", "tables_added", "tables_copied", "tables_failed", "oldest_sync_seconds",
+    "statements", "statements_refused", "statements_failed", "statements_cancelled", "rows_sent", "bytes_sent", "duration_p50_ms", "duration_p95_ms",
+    "stream_lag_bytes", "stream_lag_seconds", "rss_bytes", "heap_used_bytes", "copy_memory_bytes", "copy_memory_limit_bytes", "slot_wal_held_bytes", "disk_free_bytes", "sessions_open"];
+/** the samples' DDL: one row a sample; a level is as of `sampled_at`, a count is of the `seconds` before it */
+export const USAGE_DDL = `CREATE TABLE IF NOT EXISTS ${USAGE_TABLE} (sampled_at TIMESTAMPTZ, seconds DOUBLE, copy_bytes BIGINT, copy_rows BIGINT, ` +
+    `tables_added INTEGER, tables_copied INTEGER, tables_failed INTEGER, oldest_sync_seconds DOUBLE, statements BIGINT, statements_refused BIGINT, ` +
+    `statements_failed BIGINT, statements_cancelled BIGINT, rows_sent BIGINT, bytes_sent BIGINT, duration_p50_ms INTEGER, duration_p95_ms INTEGER, stream_lag_bytes BIGINT, ` +
+    `stream_lag_seconds DOUBLE, rss_bytes BIGINT, heap_used_bytes BIGINT, copy_memory_bytes BIGINT, copy_memory_limit_bytes BIGINT)`;
+/** the columns added since the first samples (a file a run before them made): each added where it is missing */
+export const USAGE_ADDED = ["slot_wal_held_bytes BIGINT", "disk_free_bytes BIGINT", "sessions_open INTEGER"];
+/** each copy's own bytes, every SIZE_EVERY_MS */
+export const SIZES_DDL = `CREATE TABLE IF NOT EXISTS ${SIZES_TABLE} (sampled_at TIMESTAMPTZ, table_name VARCHAR, bytes BIGINT)`;
+const noInterval = () => ({ statements: 0, outcomes: { refused: 0, failed: 0, cancelled: 0 }, rowsSent: 0, bytesSent: 0, durations: [], completed: 0 });
+/**
+ * THE COMPLETED STATEMENTS' DURATIONS ONE INTERVAL KEEPS, at most: an interval is a minute, but one whose sample cannot
+ * be taken is carried into the next (`sample`), so while samples keep failing it would otherwise keep every statement's
+ * duration for as long as they fail. Up to this many the p50 / p95 are exact; past it the interval keeps a uniform
+ * random sample of this many (a reservoir: the n-th duration replaces a kept one with probability KEPT / n), whose
+ * quantiles estimate the interval's — 100,000 numbers are ~800 KB, ~1,700 statements a second for a minute.
+ */
+export const DURATIONS_KEPT = 100_000;
+/**
+ * The q-quantile of sorted values as DuckDB's `quantile_cont` gives it (linear between the two nearest ranks), rounded
+ * as the request log's summary rounds it, or null for none: the summary's own definition of a p50 and a p95
+ * (tests/duck-usage.test.ts holds the two equal).
+ */
+export function quantileCont(sorted, q) {
+    if (sorted.length === 0)
+        return null;
+    const at = (sorted.length - 1) * q, lo = Math.floor(at), hi = Math.ceil(at);
+    return Math.round(sorted[lo] + (sorted[hi] - sorted[lo]) * (at - lo));
+}
+/**
+ * `quantileCont` over a HISTOGRAM: `[value, count]` pairs sorted by value (each count at least 0), as though every
+ * value were listed its count of times — the same ranks and the same interpolation, rounded the same way, without
+ * listing them (the request log's summary reads a week of durations this way: src/duck/requests.ts `summary`). Null
+ * for none; tests/duck-usage.test.ts holds it equal to DuckDB's `quantile_cont`.
+ */
+export function quantileContOf(histogram, q) {
+    const n = histogram.reduce((s, [, c]) => s + c, 0);
+    if (n === 0)
+        return null;
+    const at = (n - 1) * q, lo = Math.floor(at), hi = Math.ceil(at);
+    // the value at a 0-based rank: the first whose running count passes it
+    const valueAt = (rank) => {
+        let seen = 0;
+        for (const [v, c] of histogram) {
+            seen += c;
+            if (rank < seen)
+                return v;
+        }
+        return histogram[histogram.length - 1][0];
+    };
+    const a = valueAt(lo), b = hi === lo ? a : valueAt(hi);
+    return Math.round(a + (b - a) * (at - lo));
+}
+export class UsageSampler {
+    interval = noInterval();
+    totals = { completed: 0, refused: 0, failed: 0, cancelled: 0, completedMs: 0 };
+    latest = null;
+    last;
+    timer = null;
+    sampling = Promise.resolve();
+    take = null;
+    lastSized = -Infinity;
+    sizedOnce = false;
+    /** a streak of failures, said once: what failed (a sample taken, written, sized), and how many */
+    lost = new Map();
+    closed = false;
+    history;
+    log;
+    options;
+    days;
+    memoryLimitBytes;
+    sampleSeconds;
+    constructor(history, log, memoryLimitBytes, options) {
+        this.history = history;
+        this.log = log;
+        this.options = options;
+        this.days = options.days;
+        this.memoryLimitBytes = memoryLimitBytes;
+        this.sampleSeconds = (options.sampleMs ?? SAMPLE_SECONDS * 1000) / 1000;
+        this.last = this.now();
+    }
+    /** The samples of a store: their tables made, the rows past the retention deleted, and a trim every hour. Sampling
+     *  begins with `start`, once the run's facts can be read. */
+    static async open(store, log, options) {
+        const history = await History.open(store, [USAGE_DDL, ...USAGE_ADDED.map((c) => `ALTER TABLE ${USAGE_TABLE} ADD COLUMN IF NOT EXISTS ${c}`), SIZES_DDL]);
+        const u = new UsageSampler(history, log, store.memoryLimitBytes, options);
+        await history.keepTrimmed(() => u.trim(), (e) => log(`usage: deleting the samples older than ${u.days} days failed: ${e.message}`));
+        return u;
+    }
+    now() { return this.options.now?.() ?? Date.now(); }
+    /** One row of the listener's, as the request log records it: counted in the interval it ends, by the summary's rule
+     *  (a refused login is no statement; `outcomeOf`, the one outcome rule). Never throws. */
+    count(row) {
+        if (row.method === "CONNECT")
+            return;
+        const i = this.interval;
+        i.statements++;
+        const outcome = outcomeOf(row.sqlstate);
+        if (outcome === "ok") {
+            this.totals.completed++;
+            this.totals.completedMs += row.durationMs;
+        }
+        else
+            this.totals[outcome]++;
+        if (outcome === "ok") {
+            i.completed++;
+            const kept = this.options.durationsKept ?? DURATIONS_KEPT;
+            if (i.durations.length < kept)
+                i.durations.push(row.durationMs);
+            else {
+                const j = Math.floor((this.options.random?.() ?? Math.random()) * i.completed);
+                if (j < kept)
+                    i.durations[j] = row.durationMs;
+            }
+        }
+        else
+            i.outcomes[outcome]++;
+        i.rowsSent += row.rows ?? 0;
+        i.bytesSent += row.bytesSent ?? 0;
+    }
+    /** The statements since the process started, by outcome (a copy: the counter itself is never handed out). */
+    statementTotals() { return { ...this.totals }; }
+    /** The last sample's p50 and p95 of its completed statements; null before the first sample. */
+    lastDurations() { return this.latest; }
+    /** Takes a sample every interval from `take` (the run's facts), and each copy's bytes every SIZE_EVERY_MS. */
+    start(take) {
+        this.take = take;
+        this.schedule();
+    }
+    schedule() {
+        if (this.closed)
+            return;
+        // one after the other, never two at once: a sample slower than its interval delays the next
+        this.timer = setTimeout(() => { this.timer = null; this.sampling = this.sample().then(() => this.schedule()); }, this.options.sampleMs ?? SAMPLE_SECONDS * 1000);
+        this.timer.unref();
+    }
+    /** a failure of `what`, said once per streak; `ok` ends the streak, said with what it cost */
+    failure(what, e) {
+        const streak = this.lost.get(what) ?? new Streak();
+        this.lost.set(what, streak);
+        streak.failed(1, () => this.log(`usage: ${what} failed: ${e.message}; said once until one succeeds`));
+    }
+    ok(what) {
+        this.lost.get(what)?.succeeded((n) => this.log(`usage: ${what} again; ${n} time(s) it was not`));
+    }
+    /** One sample now: the run's facts, the interval's requests, the memory; written. Resolves whatever happens (a failure is said). */
+    async sample() {
+        if (this.take === null)
+            return;
+        let facts;
+        let copyMemory;
+        try {
+            facts = await this.take();
+            copyMemory = await this.copyMemory();
+        }
+        catch (e) {
+            // Tolerated, narrowly: a sample is history, and the run goes on without it. The interval's counts are KEPT for
+            // the next sample, whose `seconds` covers both, so no request goes uncounted.
+            this.failure("taking a sample", e);
+            return;
+        }
+        this.ok("taking a sample");
+        const now = this.now();
+        const i = this.interval;
+        this.interval = noInterval();
+        const seconds = (now - this.last) / 1000;
+        this.last = now;
+        const sorted = i.durations.sort((a, b) => a - b);
+        const mem = this.options.memory?.() ?? process.memoryUsage();
+        // the p50 and p95 written are the ones GET /metrics says (lastDurations): one computation
+        this.latest = { at: new Date(now).toISOString(), p50Ms: quantileCont(sorted, 0.5), p95Ms: quantileCont(sorted, 0.95) };
+        const values = [this.latest.at, seconds, facts.copyBytes, facts.totals.rows, facts.totals.added, facts.totals.copied, facts.totals.failed,
+            facts.oldestSyncSeconds, i.statements, i.outcomes.refused, i.outcomes.failed, i.outcomes.cancelled, i.rowsSent, i.bytesSent, this.latest.p50Ms, this.latest.p95Ms,
+            facts.streamLag?.bytes ?? null, facts.streamLag?.seconds ?? null, mem.rss, mem.heapUsed, copyMemory, this.memoryLimitBytes, facts.slotWalHeldBytes, facts.diskFreeBytes, facts.sessionsOpen];
+        const sql = `INSERT INTO ${USAGE_TABLE} (${COLUMNS.join(", ")}) VALUES (${COLUMNS.map((c, j) => `$${j + 1}${c === "sampled_at" ? "::TIMESTAMPTZ" : ""}`).join(", ")})`;
+        try {
+            if (this.options.insert !== undefined)
+                await this.options.insert(sql, values);
+            else
+                await this.history.writer.run(sql, values);
+            this.ok(`writing a sample to ${USAGE_TABLE}`);
+        }
+        catch (e) {
+            // Tolerated, narrowly, as the request log's: a sample that cannot be written must not stop the run. SAID.
+            this.failure(`writing a sample to ${USAGE_TABLE}`, e);
+        }
+        if (now - this.lastSized >= (this.options.sizeEveryMs ?? SIZE_EVERY_MS)) {
+            this.lastSized = now;
+            await this.size(facts.copied, now);
+        }
+    }
+    /** the copy's memory in use: DuckDB's buffer manager's counters, summed */
+    async copyMemory() {
+        const v = (await this.history.reader.runAndReadAll("SELECT sum(memory_usage_bytes) FROM duckdb_memory()")).getRowsJS()[0]?.[0];
+        return v === null || v === undefined ? null : Number(v);
+    }
+    /** Each served copy's own bytes, written as one batch (see the module's comment for how, and what it costs). */
+    async size(copied, now) {
+        const t0 = performance.now();
+        const rows = [];
+        try {
+            const want = new Set(copied);
+            const tables = (await this.history.reader.runAndReadAll(`SELECT schema_name, table_name FROM duckdb_tables() WHERE NOT internal AND database_name = current_database() AND schema_name <> '${MASON_SCHEMA}'`))
+                .getRowsJS();
+            // THE COPY'S OWN database: pragma_database_size() has a row for every attached one, and the in-memory catalog the
+            // visible names live in (src/duck/kept.ts) has block_size 0 — read first, it made every copy "0 B"
+            const block = Number((await this.history.reader.runAndReadAll("SELECT block_size FROM pragma_database_size() WHERE database_name = current_database()")).getRowsJS()[0][0]);
+            for (const [schema, table] of tables) {
+                const name = `${schema}.${table}`;
+                if (!want.has(name))
+                    continue;
+                const q = `${quote(schema)}.${quote(table)}`;
+                let blocks;
+                try {
+                    blocks = Number((await this.history.reader.runAndReadAll(`WITH s AS (SELECT block_id, additional_block_ids FROM pragma_storage_info($1) WHERE persistent), ` +
+                        `b AS (SELECT block_id FROM s WHERE block_id >= 0 UNION SELECT unnest(additional_block_ids) FROM s) SELECT count(*) FROM b`, [q])).getRowsJS()[0][0]);
+                }
+                catch (e) {
+                    // a copy dropped since the list was read (mason remove; a copy swapped in again): not sized this time, by design.
+                    // Anything else is the sizing's failure, said
+                    if (/does not exist/i.test(e.message))
+                        continue;
+                    throw e;
+                }
+                // NO BLOCK of its own yet is not 0 bytes: its rows are still in the write-ahead log (a small copy is written to
+                // the file only at a checkpoint), so it is not measured this time — a page said "0.0 KiB" for a 5,000-row copy
+                if (blocks === 0)
+                    continue;
+                rows.push([name, blocks * block]);
+            }
+            if (rows.length > 0) {
+                const sql = `INSERT INTO ${SIZES_TABLE} (sampled_at, table_name, bytes) VALUES ` + rows.map((_, i) => `($1::TIMESTAMPTZ, $${i * 2 + 2}, $${i * 2 + 3})`).join(", ");
+                const values = [new Date(now).toISOString(), ...rows.flat()];
+                if (this.options.insert !== undefined)
+                    await this.options.insert(sql, values);
+                else
+                    await this.history.writer.run(sql, values);
+            }
+        }
+        catch (e) {
+            this.failure("measuring each synced table's size", e);
+            return;
+        }
+        this.ok("measuring each synced table's size");
+        // the first measure of a run is said with its time: what it costs on this machine's copies, read from the log
+        if (!this.sizedOnce) {
+            this.sizedOnce = true;
+            this.log(`usage: measured ${rows.length} synced tables' own size in ${Math.round(performance.now() - t0)} ms (every ${Math.round((this.options.sizeEveryMs ?? SIZE_EVERY_MS) / 60_000)} min)`);
+        }
+    }
+    /** Deletes the samples and sizes older than the retention. */
+    async trim() {
+        await this.history.writer.run(`DELETE FROM ${USAGE_TABLE} WHERE sampled_at < now() - INTERVAL (${this.days}) DAY`);
+        await this.history.writer.run(`DELETE FROM ${SIZES_TABLE} WHERE sampled_at < now() - INTERVAL (${this.days}) DAY`);
+    }
+    /**
+     * The samples from `since` until `until`, oldest first; with `stepSeconds`, re-bucketed in ONE query: buckets of that
+     * many seconds from `since`, each count summed and each level at its highest (the disk free at its lowest). Committed rows only.
+     */
+    async samples(o) {
+        const values = [o.since, o.until];
+        const range = `sampled_at >= $1::TIMESTAMPTZ AND sampled_at < $2::TIMESTAMPTZ`;
+        let sql;
+        if (o.stepSeconds === undefined) {
+            sql = `SELECT epoch_ms(sampled_at) AS at, ${COLUMNS.slice(1).join(", ")} FROM ${USAGE_TABLE} WHERE ${range} ORDER BY sampled_at`;
+        }
+        else {
+            values.push(o.stepSeconds);
+            const sum = new Set(["seconds", "statements", "statements_refused", "statements_failed", "statements_cancelled", "rows_sent", "bytes_sent"]);
+            sql = `SELECT epoch_ms(time_bucket(to_seconds($3::BIGINT), sampled_at, $1::TIMESTAMPTZ)) AS at, ` +
+                COLUMNS.slice(1).map((c) => `${sum.has(c) ? "sum" : c === "disk_free_bytes" ? "min" : "max"}(${c}) AS ${c}`).join(", ") +
+                ` FROM ${USAGE_TABLE} WHERE ${range} GROUP BY 1 ORDER BY 1`;
+        }
+        const rows = (await this.history.reader.runAndReadAll(sql, values)).getRowObjectsJson();
+        return rows.map(sampleOf);
+    }
+    /** Each copy's own bytes as last measured within the retention, by table. */
+    async copySizes() {
+        const rows = (await this.history.reader.runAndReadAll(`SELECT table_name, arg_max(bytes, sampled_at) AS bytes, epoch_ms(max(sampled_at)) AS at FROM ${SIZES_TABLE} WHERE bytes > 0 GROUP BY table_name ORDER BY table_name`))
+            .getRowsJS();
+        return rows.map(([table, bytes, at]) => ({ table, bytes: Number(bytes), at: new Date(Number(at)).toISOString() }));
+    }
+    /** Stops sampling, waits for a sample in flight, and closes the samples' connections (mason stop). */
+    async close() {
+        this.closed = true;
+        if (this.timer !== null) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+        this.history.stopTrimming();
+        await this.sampling;
+        this.history.closeConnections();
+    }
+}
+const quote = (name) => `"${name.replaceAll('"', '""')}"`;
+const num = (v) => (v === null || v === undefined ? null : Number(v));
+function sampleOf(r) {
+    return {
+        at: new Date(Number(r.at)).toISOString(), seconds: Number(r.seconds), copyBytes: num(r.copy_bytes), copyRows: Number(r.copy_rows ?? 0),
+        tablesAdded: Number(r.tables_added ?? 0), tablesCopied: Number(r.tables_copied ?? 0), tablesFailed: Number(r.tables_failed ?? 0), oldestSyncSeconds: num(r.oldest_sync_seconds),
+        statements: Number(r.statements ?? 0), statementsRefused: Number(r.statements_refused ?? 0), statementsFailed: Number(r.statements_failed ?? 0),
+        statementsCancelled: Number(r.statements_cancelled ?? 0), rowsSent: Number(r.rows_sent ?? 0), bytesSent: Number(r.bytes_sent ?? 0), durationP50Ms: num(r.duration_p50_ms), durationP95Ms: num(r.duration_p95_ms),
+        streamLagBytes: num(r.stream_lag_bytes), streamLagSeconds: num(r.stream_lag_seconds),
+        rssBytes: Number(r.rss_bytes), heapUsedBytes: Number(r.heap_used_bytes), copyMemoryBytes: num(r.copy_memory_bytes), copyMemoryLimitBytes: Number(r.copy_memory_limit_bytes),
+        slotWalHeldBytes: num(r.slot_wal_held_bytes), diskFreeBytes: num(r.disk_free_bytes), sessionsOpen: num(r.sessions_open),
+    };
+}

package/dist/src/duck/users.js

@@ -11,10 +11,14 @@
  * statement of its session is attributed to that user, and rotating the API key removes every user at once (no MAC
  * verifies under the new key). It never collides with a token: a token's MAC is 43 base64url characters.
  *
- * THE LIST OF LIVE USERS is a FILE in the state folder (`.mason/users.json`), never a DuckDB table: `mason user …` runs
+ * THE LIST OF USERS is a FILE in the state folder (`.mason/users.json`), never a DuckDB table: `mason user …` runs
  * as a second process beside the run, which holds DuckDB's file lock. It holds each user's name, id, when it was added
  * and the API key's fingerprint at the time (`key`, a MAC of a constant: no secret, it says which key a user was added
- * under) — never a password. A change is made under a lock file (`users.json.lock`, O_EXCL) and written beside and
+ * under) — never a password. A removed user STAYS in it with `removedAt` (so a list can show who was removed, and when):
+ * only the entries without one are live, and the listener reads only those. A DISABLED user stays too, with
+ * `disabledAt` (`mason user disable`): its password still verifies, but its login is refused as Postgres refuses a role
+ * without LOGIN (28000, "is not permitted to log in") and its sessions end at their next statement, until `mason user
+ * enable` clears it — the same id, so the same password connects again. A change is made under a lock file (`users.json.lock`, O_EXCL) and written beside and
  * renamed, so a reader sees the old list or the new one, never half of either, and two `mason user add` at once cannot
  * both win a name. The listener RE-READS it (`LiveUsers`: a stat at every connection and before every statement of a
  * user's session, the file read again when its inode, size or mtime moved), so a removal refuses the next connection
@@ -24,13 +28,27 @@
 import * as crypto from "node:crypto";
 import * as fs from "node:fs";
 import * as path from "node:path";
-import { ApiError } from "../api/error.js";
+import { ApiError, USER_DISABLED } from "../api/error.js";
 import { tokenProblem } from "./token.js";
+import { ATTRIBUTE_NAME_RE } from "./row-filter.js";
 export const USERS_FILE = "users.json";
 /** A user's name: what a BI tool's form calls the user (Postgres's NAMEDATALEN, 63), lower case so a form that
  *  capitalises it is told so, never another user. */
 export const USER_NAME_RE = /^[a-z0-9][a-z0-9_.-]{0,62}$/;
 export const USER_NAME_RULE = "1 to 63 lower case letters, digits, \"_\", \".\" or \"-\", starting with a letter or digit (e.g. metabase)";
+/** a user's attributes as the file keeps them: a name each, a string or a list of strings */
+function attributesValid(v) {
+    // a NUL ends the statement it is bound into, wherever it is: every statement of the user would fail as a syntax error
+    const value = (x) => typeof x === "string" && !x.includes("\0");
+    return typeof v === "object" && v !== null && !Array.isArray(v) &&
+        Object.entries(v).every(([k, x]) => ATTRIBUTE_NAME_RE.test(k) && (value(x) || (Array.isArray(x) && x.every(value))));
+}
+/** Whether a user has not been removed (it may still not connect: see `underKey`). */
+export const isActive = (u) => u.removedAt === undefined;
+/** Whether a user is disabled (`mason user disable`): kept, its password unchanged, its logins refused until enabled. */
+export const isDisabled = (u) => u.disabledAt !== undefined;
+/** A role's name as the users file may carry it (src/cli/roles.ts ROLE_NAME_RE, which the roles file holds it to). */
+const ROLE_RE = /^[a-z0-9][a-z0-9_-]{0,62}$/;
 const PASSWORD_RE = /^mason_([0-9a-f]{16})_([0-9a-f]{32})$/;
 const mac = (apiKey, name, id) => crypto.createHmac("sha256", apiKey).update(`mason-user:${name}:${id}`).digest("hex").slice(0, 32);
 /** Which API key a user was added under, without the key: a MAC of a constant, 16 hex digits. */
@@ -71,10 +89,19 @@ function parseUsers(file, text) {
     for (const [i, u] of list.entries()) {
         const o = u;
         if (typeof o !== "object" || o === null || typeof o.name !== "string" || !USER_NAME_RE.test(o.name) || typeof o.id !== "string" || !/^[0-9a-f]{16}$/.test(o.id) ||
-            typeof o.addedAt !== "string" || typeof o.key !== "string") {
-            return { problem: `${file}: users[${i}] is not { name, id, addedAt, key } as mason user add writes it` };
+            typeof o.addedAt !== "string" || typeof o.key !== "string" || (o.role !== undefined && (typeof o.role !== "string" || !ROLE_RE.test(o.role))) ||
+            (o.attributes !== undefined && !attributesValid(o.attributes))) {
+            return { problem: `${file}: users[${i}] is not { name, id, addedAt, key, role?, attributes?[, removedAt][, disabledAt] } as mason user add writes it` };
+        }
+        if (o.removedAt !== undefined && typeof o.removedAt !== "string") {
+            return { problem: `${file}: users[${i}].removedAt is not a timestamp as mason user remove writes it` };
+        }
+        if (o.disabledAt !== undefined && typeof o.disabledAt !== "string") {
+            return { problem: `${file}: users[${i}].disabledAt is not a timestamp as mason user disable writes it` };
         }
-        users.push({ name: o.name, id: o.id, addedAt: o.addedAt, key: o.key });
+        users.push({ name: o.name, id: o.id, addedAt: o.addedAt, key: o.key, ...(typeof o.role === "string" ? { role: o.role } : {}),
+            ...(attributesValid(o.attributes) ? { attributes: o.attributes } : {}),
+            ...(o.removedAt !== undefined ? { removedAt: o.removedAt } : {}), ...(o.disabledAt !== undefined ? { disabledAt: o.disabledAt } : {}) });
     }
     return { users };
 }
@@ -140,8 +167,8 @@ export function changeUsers(file, change) {
 }
 // ------------------------------------------------------------------ the listener's view
 /** The ApiError a refused user login carries: the user it asked for, so the request log names who still tries. */
-function refusal(status, message, user) {
-    const e = new ApiError(status, message, status === 503 ? "users_unreadable" : "invalid_user");
+function refusal(status, message, user, code = status === 503 ? "users_unreadable" : "invalid_user") {
+    const e = new ApiError(status, message, code);
     e.userName = user;
     return e;
 }
@@ -156,7 +183,7 @@ export class LiveUsers {
     seen = null;
     read = { users: [] };
     constructor(file, apiKey) { this.file = file; this.apiKey = apiKey; }
-    /** the users now; why not, when the file cannot be read */
+    /** the users not removed now (disabled ones included: their login is refused by name); why not, when the file cannot be read */
     now() {
         let stamp;
         try {
@@ -169,7 +196,9 @@ export class LiveUsers {
             stamp = "none";
         }
         if (stamp !== this.seen || "problem" in this.read) {
-            this.read = readUsers(this.file);
+            const r = readUsers(this.file);
+            // a removed user is kept in the file for the list: it is no user to the listener
+            this.read = "problem" in r ? r : { users: r.users.filter(isActive) };
             this.seen = stamp;
         }
         return this.read;
@@ -197,19 +226,39 @@ export class LiveUsers {
         // is the login — to anyone holding only the password (found on the test server)
         if (found.name !== user)
             throw refusal(401, "not a password of this project's users: mason user list shows them", user);
-        return { filters: [], subject: undefined, user: found.name, jti: found.id };
+        // disabled: refused only once its password verified (so the state is told to the holder of the login alone), in
+        // Postgres's words for a role without LOGIN, 28000 (src/api/postgres/server.ts reads the code)
+        if (isDisabled(found))
+            throw refusal(401, disabledMessage(found.name), user, USER_DISABLED);
+        return { filters: [], subject: undefined, user: found.name, jti: found.id, ...(found.role !== undefined ? { role: found.role } : {}), ...attributesOf(found) };
     }
-    /** A held session's user, asked again before a statement: throws as `login` does when it is no longer live. */
+    /**
+     * A held session's user, asked again before a statement: throws as `login` does when it is no longer live; else the
+     * token as the file has the user NOW — its role as `mason user add --role` wrote it, so a role given or taken
+     * applies at the user's next statement, without a restart.
+     */
     recheck(token) {
         if (token.user === undefined)
-            return;
+            return token;
         const now = this.now();
         if ("problem" in now)
             throw refusal(503, `the list of users cannot be read: ${now.problem}`, token.user);
-        if (!now.users.some((u) => u.id === token.jti && u.name === token.user))
+        const found = now.users.find((u) => u.id === token.jti && u.name === token.user);
+        if (found === undefined)
             throw refusal(401, `the user "${token.user}" was removed`, token.user);
+        if (isDisabled(found))
+            throw refusal(401, `the user "${token.user}" was disabled`, token.user, USER_DISABLED);
+        if (found.role === token.role && JSON.stringify(found.attributes ?? null) === JSON.stringify(token.attributes ?? null))
+            return token;
+        const { role: _was, attributes: _had, ...rest } = token;
+        return { ...rest, ...(found.role !== undefined ? { role: found.role } : {}), ...attributesOf(found) };
     }
 }
+/** a user's attributes as its session's token carries them (src/api/postgres/server.ts PgToken) */
+const attributesOf = (u) => u.attributes !== undefined && Object.keys(u.attributes).length > 0 ? { attributes: u.attributes } : {};
+/** A disabled user's refused login: Postgres's sentence for a role without LOGIN, and it alone — why is its DETAIL,
+ *  the way out its HINT (src/api/postgres/server.ts), as Postgres splits them. */
+export const disabledMessage = (name) => `role "${name}" is not permitted to log in`;
 /**
  * HOW THE DUCKDB RUN'S LISTENER TAKES A PASSWORD (src/cli/dev.ts; the tests wire the same): a user's password (its shape)
  * is the users file's to judge, and its session is asked again before every statement (a stat); anything else is read as a
@@ -225,7 +274,7 @@ export function listenerLogins(users, apiKey) {
                 throw new Error(problem);
             return { filters: [], subject: undefined };
         },
-        recheck: async (t) => { users.recheck(t); },
+        recheck: async (t) => users.recheck(t),
         recheckMs: 0,
     };
 }

package/dist/src/duck/verified.js

@@ -242,6 +242,7 @@ export const VERIFIED = new Map([
     ["field:SUBLINK.subLinkType=0", "EXISTS (subquery)"],
     ["field:SUBLINK.subLinkType=2", "IN / op ANY (subquery)"],
     ["field:SUBLINK.subLinkType=4", "a scalar subquery"],
+    ["field:SUBLINK.subLinkType=6", "ARRAY(subquery)"],
     ["field:WINDOWCLAUSE.frameOptions=1058", "a window frame: the default window frame"],
     ["field:WINDOWCLAUSE.frameOptions=1059", "a window frame: RANGE UNBOUNDED PRECEDING"],
     ["field:WINDOWCLAUSE.frameOptions=1061", "a window frame: ROWS UNBOUNDED PRECEDING"],
@@ -358,6 +359,7 @@ export const VERIFIED = new Map([
     ["function:2059", "the function age(timestamp without time zone)"],
     ["function:2069", "the function timezone(text,timestamp without time zone)"],
     ["function:2073", "the function \"substring\"(text,text)"],
+    ["function:2077", "the function current_setting(text)"],
     ["function:2087", "the function replace(text,text,text)"],
     ["function:2088", "the function split_part(text,text,integer)"],
     ["function:2089", "the function to_hex(integer)"],
@@ -413,6 +415,7 @@ export const VERIFIED = new Map([
     ["function:3271", "the function jsonb_build_array(\"any\")"],
     ["function:3273", "the function jsonb_build_object(\"any\")"],
     ["function:3277", "the function array_position(anycompatiblearray,anycompatible)"],
+    ["function:3294", "the function current_setting(text,boolean)"],
     ["function:3396", "the function regexp_match(text,text)"],
     ["function:3464", "the function make_interval(integer,integer,integer,integer,integer,integer,double precision)"],
     ["function:3465", "the function jsonb_array_elements_text(jsonb)"],
@@ -500,6 +503,7 @@ export const VERIFIED = new Map([
     ["function:941", "the function mod(integer,integer)"],
     ["function:947", "the function mod(bigint,bigint)"],
     ["function:ext:current_setting(text)", "the function current_setting(text) (an extension's)"],
+    ["function:ext:current_setting(text,boolean)", "the function current_setting(text,boolean) (an extension's)"],
     ["node:AGGREF", "an aggregate"],
     ["node:ALIAS", "an alias"],
     ["node:ARRAYCOERCEEXPR", "a cast of an array's elements"],
@@ -736,6 +740,11 @@ export const VERIFIED = new Map([
     ["operator:3212", "the operator ->(jsonb,integer)"],
     ["operator:3213", "the operator #>(jsonb,text[])"],
     ["operator:3240", "the operator =(jsonb,jsonb)"],
+    ["operator:3241", "the operator <>(jsonb,jsonb)"],
+    ["operator:3242", "the operator <(jsonb,jsonb)"],
+    ["operator:3243", "the operator >(jsonb,jsonb)"],
+    ["operator:3244", "the operator <=(jsonb,jsonb)"],
+    ["operator:3245", "the operator >=(jsonb,jsonb)"],
     ["operator:3246", "the operator @>(jsonb,jsonb)"],
     ["operator:3247", "the operator ?(jsonb,text)"],
     ["operator:3248", "the operator ?|(jsonb,text[])"],

package/dist/src/duck/views.js

@@ -15,10 +15,9 @@
  * view's name.
  */
 import { deparseSync } from "pgsql-deparser";
-import { StatementError } from "../api/postgres/server.js";
+import { gapRefusal } from "../api/postgres/server.js";
 import { parseSql } from "../api/postgres/parser.js";
 import { catalogRows } from "./analyze.js";
-import { gapMark } from "./gaps.js";
 import { treeDifference } from "./rewrite.js";
 /** views read through views read through views…: past this depth the statement is refused (a definition is never
  *  recursive, so only a deep stack of views reaches it) */
@@ -28,7 +27,7 @@ const quote = (name) => `"${name.replace(/"/g, '""')}"`;
 const literal = (s) => `'${s.replace(/'/g, "''")}'`;
 const sval = (n) => isObject(n) && isObject(n.String) && typeof n.String.sval === "string" ? n.String.sval : null;
 export function viewRefused(message, hint) {
-    return new StatementError({ code: "0A000", message, hint, detail: `${gapMark("view-forms")} (docs/SQL-SUPPORT.md)` });
+    return gapRefusal("view-forms", message, hint);
 }
 /** the views Postgres's analysis read: every range-table entry of a relation whose kind is a view, by OID */
 export function viewsRead(tree) {