@haystackeditor/cli 0.17.1 → 0.19.0

package/dist/commands/db-profile-database.js

@@ -0,0 +1,289 @@
+/**
+ * The profiler's only door into the customer's database.
+ *
+ * Every statement runs in its own READ ONLY transaction with a statement
+ * timeout and a lock timeout, and every one of those transactions reads the
+ * same REPEATABLE READ snapshot (exported by one more transaction that stays
+ * open for the run, `pg_export_snapshot`), on a connection that starts with
+ * `default_transaction_read_only = on` (a startup parameter, so it applies to
+ * the profiler's own server session and never to a pooled connection another
+ * client reuses), and both read-only settings are checked before the
+ * statement runs. The transaction is always rolled back: nothing the profiler
+ * does is ever meant to be kept. Only single SELECT / WITH statements are
+ * accepted, so a bug that built anything else fails here instead of running.
+ *
+ * It works on a read replica (hot standby) too. Nothing here writes, locks
+ * for write or creates a temporary table, and the isolation level is named
+ * (REPEATABLE READ) rather than inherited, because a standby refuses every
+ * statement when its default_transaction_isolation is serializable.
+ */
+import pg from 'pg';
+import { parse as parseConnectionString } from 'pg-connection-string';
+/** A table lock the profiler cannot get within this time fails the run rather
+ * than queueing behind (and in front of) a migration's exclusive lock. */
+export const LOCK_TIMEOUT_MS = 5_000;
+const READ_STATEMENT = /^\s*(SELECT|WITH)\b/iu;
+export function assertReadStatement(sql, label) {
+    if (!READ_STATEMENT.test(sql))
+        throw new Error(`${label}: refusing to run a statement that is not a SELECT.`);
+    if (sql.includes(';'))
+        throw new Error(`${label}: refusing to run more than one statement at once.`);
+}
+function errorText(error) {
+    if (error instanceof Error) {
+        const code = error.code;
+        return typeof code === 'string' && /^[0-9A-Z]{5}$/u.test(code) ? `${error.message} (SQLSTATE ${code})` : error.message;
+    }
+    return String(error);
+}
+/** What to change when a read replica cancels a query because replaying the
+ * primary's changes (a vacuum removing rows the query still reads) could not
+ * wait for it any longer. */
+export const RECOVERY_CONFLICT_HINT = 'The read replica canceled the query to keep replaying the primary; turn on hot_standby_feedback on the replica, or raise its max_standby_streaming_delay (on RDS and Cloud SQL, in its parameter group or flags) to longer than --statement-timeout, then run again.';
+function isRecoveryConflict(error) {
+    if (!(error instanceof Error))
+        return false;
+    // A READ COMMITTED read-only statement meets serialization_failure only as
+    // a recovery conflict; the message covers servers with English messages
+    // that report it under another code (a buffer-pin deadlock is 40P01).
+    return error.code === '40001' || /conflict with recovery/iu.test(error.message);
+}
+const READ_ONLY_STARTUP = '-c default_transaction_read_only=on';
+/**
+ * The connection settings of a connection string, parsed by pg-connection-string
+ * (the parser pg itself runs on a `connectionString`), so every URL pg accepts
+ * works, a Unix socket too (`postgresql://user@/db?host=/var/run/postgresql`,
+ * which WHATWG `URL` refuses), with `-c default_transaction_read_only=on`
+ * added to the `options` startup parameter (after any options it already
+ * has). The pool is given these settings, never the string: pg would let the
+ * string override them. A `connectionString` inside the string would do the
+ * same, so it is refused. The string is never echoed: it may hold a password.
+ */
+export function readOnlyConnectionSettings(connectionString) {
+    let parsed;
+    try {
+        parsed = parseConnectionString(connectionString);
+    }
+    catch (error) {
+        throw new Error(`The connection string is not a valid postgres:// URL (${error instanceof Error ? error.name : 'parse error'}).`);
+    }
+    if (parsed.connectionString !== undefined) {
+        throw new Error('The connection string sets connectionString, which would replace the read-only startup option; remove it.');
+    }
+    const existing = parsed.options;
+    return {
+        ...parsed,
+        options: existing === undefined || existing.trim() === '' ? READ_ONLY_STARTUP : `${existing} ${READ_ONLY_STARTUP}`,
+    };
+}
+/** The first server version (17) with the transaction_timeout setting. */
+const TRANSACTION_TIMEOUT_VERSION_NUM = 170_000;
+/** A `pg_export_snapshot()` identifier, checked before it is embedded in SQL. */
+const SNAPSHOT_ID = /^[0-9A-Fa-f]+(-[0-9A-Fa-f]+)+$/u;
+export class ReadOnlyDatabase {
+    concurrency;
+    pool;
+    statementTimeoutMs;
+    poolError = null;
+    /** The transaction whose snapshot every query reads; opened by the first query. */
+    snapshot = null;
+    snapshotError = null;
+    /** Pooled sessions whose transaction timeouts are already off (see `connect`). */
+    untimed = new WeakSet();
+    constructor(connectionString, concurrency, statementTimeoutMs) {
+        this.concurrency = concurrency;
+        this.statementTimeoutMs = statementTimeoutMs;
+        this.pool = new pg.Pool({
+            // The string's own application_name wins, as it did when pg parsed it.
+            application_name: 'haystack-db-profiler',
+            ...readOnlyConnectionSettings(connectionString),
+            // Pool and timeout settings come last: nothing in the string changes them.
+            // One more than the queries in flight: the connection holding the snapshot.
+            max: concurrency + 1,
+            // Never keep a customer connection open longer than the run needs.
+            idleTimeoutMillis: 10_000,
+            connectionTimeoutMillis: 30_000,
+        });
+        this.pool.on('error', error => {
+            // An idle connection died (server restart, network). Remember it; the
+            // next query reports it instead of the process crashing.
+            this.poolError = error;
+        });
+    }
+    async connect(label) {
+        if (this.poolError)
+            throw new Error(`${label}: the database connection failed earlier: ${errorText(this.poolError)}`);
+        let client;
+        try {
+            client = await this.pool.connect();
+        }
+        catch (error) {
+            const text = errorText(error);
+            const pooler = /unsupported startup parameter/iu.test(text)
+                ? ' The connection goes through a pooler that refuses startup options (PgBouncer does by default); point the profiler at Postgres directly.'
+                : '';
+            throw new Error(`${label}: could not connect to the database: ${text}.${pooler}`);
+        }
+        if (this.untimed.has(client))
+            return client;
+        try {
+            // The snapshot's transaction sits idle for the whole run, and a query
+            // may run longer than a hosted default allows a transaction: a role or
+            // database default for idle_in_transaction_session_timeout (or, from
+            // Postgres 17, transaction_timeout) would end it and the run with it.
+            // These are the profiler's own sessions, and any role may set both.
+            // SHOW and SET take no snapshot, so they run outside a transaction even
+            // on a standby whose default isolation is serializable.
+            const [{ server_version_num: num }] = (await client.query('SHOW server_version_num')).rows;
+            await client.query('SET idle_in_transaction_session_timeout = 0');
+            if (Number(num) >= TRANSACTION_TIMEOUT_VERSION_NUM)
+                await client.query('SET transaction_timeout = 0');
+        }
+        catch (error) {
+            client.release(error instanceof Error ? error : new Error(String(error)));
+            throw new Error(`${label}: could not turn off the session's transaction timeouts: ${errorText(error)}`);
+        }
+        this.untimed.add(client);
+        return client;
+    }
+    /** Settings and read-only checks every transaction starts with, after any
+     * SET TRANSACTION SNAPSHOT (which must come first). */
+    async prepare(client) {
+        // SET LOCAL lasts until the rollback, so a pooled connection (including
+        // behind PgBouncer in transaction mode) is left unchanged.
+        await client.query(`SET LOCAL statement_timeout = ${Math.round(this.statementTimeoutMs)}`);
+        await client.query(`SET LOCAL lock_timeout = ${LOCK_TIMEOUT_MS}`);
+        await client.query('SET LOCAL standard_conforming_strings = on');
+        await client.query('SET LOCAL search_path = pg_catalog, pg_temp');
+        const state = await client.query(`SELECT current_setting('transaction_read_only') AS read_only, current_setting('default_transaction_read_only') AS default_read_only`);
+        if (state.rows[0]?.read_only !== 'on') {
+            throw new Error('the transaction is not read only; refusing to run');
+        }
+        if (state.rows[0]?.default_read_only !== 'on') {
+            throw new Error('the session did not start with default_transaction_read_only = on (a pooler may have dropped the startup option); refusing to run');
+        }
+    }
+    /**
+     * Every query reads the database as of one moment. The first query opens a
+     * REPEATABLE READ, READ ONLY transaction on a connection of its own,
+     * exports its snapshot and keeps it open until `close`; every query's own
+     * transaction imports that snapshot. Otherwise a row written between two
+     * queries could make one count exceed another (a share above 1).
+     */
+    exportedSnapshot() {
+        this.snapshot ??= (async () => {
+            const label = 'read-only snapshot';
+            const client = await this.connect(label);
+            // A checked-out connection that dies emits here, not on the pool.
+            client.on('error', error => { this.snapshotError = error; });
+            try {
+                await client.query('BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY');
+                await this.prepare(client);
+                const { rows } = await client.query('SELECT pg_export_snapshot() AS id');
+                const id = rows[0]?.id;
+                if (typeof id !== 'string' || !SNAPSHOT_ID.test(id))
+                    throw new Error('the server returned no snapshot identifier');
+                return { client, id };
+            }
+            catch (error) {
+                client.release(error instanceof Error ? error : new Error(String(error)));
+                throw new Error(`${label}: ${errorText(error)}${isRecoveryConflict(error) ? ` ${RECOVERY_CONFLICT_HINT}` : ''}`);
+            }
+        })();
+        return this.snapshot;
+    }
+    async query(label, sql, params = []) {
+        assertReadStatement(sql, label);
+        const snapshot = await this.exportedSnapshot();
+        if (this.snapshotError) {
+            throw new Error(`${label}: the connection holding the profiler's snapshot failed: ${errorText(this.snapshotError)}`);
+        }
+        const client = await this.connect(label);
+        let failure = null;
+        let inTransaction = false;
+        try {
+            // The isolation level is named, never inherited: a standby refuses
+            // every transaction whose default is serializable.
+            await client.query('BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY');
+            inTransaction = true;
+            await client.query(`SET TRANSACTION SNAPSHOT '${snapshot.id}'`);
+            await this.prepare(client);
+            const result = await client.query(sql, params);
+            return result.rows;
+        }
+        catch (error) {
+            failure = error;
+            throw new Error(`${label}: ${errorText(error)}${isRecoveryConflict(error) ? ` ${RECOVERY_CONFLICT_HINT}` : ''}`);
+        }
+        finally {
+            let rollbackFailure = null;
+            if (inTransaction) {
+                try {
+                    await client.query('ROLLBACK');
+                }
+                catch (error) {
+                    rollbackFailure = error;
+                }
+            }
+            // A connection whose rollback failed is in an unknown state: destroy it.
+            const broken = rollbackFailure ?? (inTransaction ? null : failure);
+            client.release(broken instanceof Error ? broken : broken === null ? undefined : new Error(String(broken)));
+            if (rollbackFailure !== null && failure === null) {
+                // eslint-disable-next-line no-unsafe-finally -- the rollback failing is itself the failure to report
+                throw new Error(`${label}: rolling back the read-only transaction failed: ${errorText(rollbackFailure)}`);
+            }
+        }
+    }
+    async close() {
+        let holder = null;
+        if (this.snapshot !== null) {
+            // A snapshot that failed to open was already reported by the query that
+            // opened it; there is nothing to roll back then.
+            holder = await this.snapshot.then(value => value, () => null);
+        }
+        if (holder !== null && this.snapshotError !== null) {
+            // The server already ended that session (and its transaction); the
+            // query that found out reported it, so there is nothing to roll back
+            // and no second error to put in front of the first.
+            holder.client.release(this.snapshotError);
+        }
+        else if (holder !== null) {
+            try {
+                await holder.client.query('ROLLBACK');
+                holder.client.release();
+            }
+            catch (error) {
+                holder.client.release(error instanceof Error ? error : new Error(String(error)));
+                await this.pool.end();
+                throw new Error(`read-only snapshot: rolling back failed: ${errorText(error)}`);
+            }
+        }
+        await this.pool.end();
+    }
+}
+/** Run `work` over `items` with at most `limit` in flight. After the first
+ * failure no new item starts; in-flight items finish, then the first failure
+ * is thrown. */
+export async function runBounded(items, limit, work) {
+    let next = 0;
+    const failures = [];
+    const worker = async () => {
+        while (failures.length === 0 && next < items.length) {
+            const item = items[next];
+            next += 1;
+            try {
+                await work(item);
+            }
+            catch (error) {
+                failures.push(error);
+            }
+        }
+    };
+    await Promise.all(Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, () => worker()));
+    if (failures.length === 1)
+        throw failures[0];
+    if (failures.length > 1) {
+        const first = failures[0] instanceof Error ? failures[0].message : String(failures[0]);
+        throw new Error(`${first} (and ${failures.length - 1} other in-flight ${failures.length === 2 ? 'query' : 'queries'} also failed)`);
+    }
+}

package/dist/commands/db-profile-distinct.js

@@ -0,0 +1,97 @@
+/**
+ * Distinct-value estimates for sampled tables, kept free of database access
+ * so the arithmetic can be run on its own.
+ *
+ * A `TABLESAMPLE SYSTEM` sample reads whole pages, so rows that sit together
+ * (an order's line items, a user's rows written in one burst) arrive
+ * together. The estimate therefore counts, for each distinct value, the
+ * sampled PAGES holding it rather than its rows: a value clustered on one page
+ * is then seen once, which is what it is to a page sample. Each page is in the
+ * sample with probability q, so these page counts are a Bernoulli(q) sample of
+ * each value's pages, and the classical estimators apply to them. Only the
+ * summary (how many values were found on exactly j pages, for each j) leaves
+ * the database.
+ *
+ * The unseen values are the geometric mean of two estimates that err in
+ * opposite directions on skewed data: the adaptive estimator (AE) of Charikar,
+ * Chaudhuri, Motwani and Narasayya ("Towards estimation error guarantees for
+ * distinct values", PODS 2000), which reads low on heavy tails, and Shlosser's
+ * estimator, which Haas et al. recommend for skew and which reads high there.
+ * The result is held between two bounds of Bernoulli page sampling:
+ *   - at most f1 (1 - q) / q unseen values, reached when every value lives on
+ *     exactly one page;
+ *   - at least the Chao1 bound for sampling without replacement (Chao and
+ *     Lin, 2012), f1^2 / (2 f2 n / (n - 1) + f1 q / (1 - q)).
+ * Measured against exact counts on a skewed (Zipf) 2 GB dataset, at page
+ * samples from 0.24% to 66%, it was the closest of the estimators tried (Duj1,
+ * which the planner uses; Duj2; GEE; Chao1; Chao-Lee; Shlosser; AE; a
+ * unique-value mixture), and counting pages beat counting rows for each. No
+ * sample estimate is exact. There, heavy-tailed columns read 1.1x to 1.6x
+ * under a 2.4% sample and up to 2.3x under 1%, and the worst column read
+ * 0.17x (a JSON column of mostly unique values under a sample below 1%);
+ * the planner's estimate read 0.08x and 0.02x on those columns. Raising
+ * --scan-max-mb narrows it; a table read in full is counted exactly.
+ */
+function tally(fingerprint, term) {
+    return fingerprint.reduce((sum, entry) => sum + entry.values * term(entry.pages), 0);
+}
+function valuesOn(fingerprint, pages) {
+    return fingerprint.find(entry => entry.pages === pages)?.values ?? 0;
+}
+/** AE's unseen values (m - f1 - f2), at most `most`. */
+function adaptiveUnseen(fingerprint, most) {
+    const f1 = valuesOn(fingerprint, 1);
+    const f2 = valuesOn(fingerprint, 2);
+    const rate = f1 + 2 * f2;
+    const tailWeight = tally(fingerprint, pages => (pages >= 3 ? Math.exp(-pages) : 0));
+    const tailPages = tally(fingerprint, pages => (pages >= 3 ? pages * Math.exp(-pages) : 0));
+    // m is AE's count of low-frequency values in the table. It solves g(m) = 0,
+    // and g(f1 + f2) < 0 whenever f1 > 0.
+    const g = (m) => {
+        const seen = Math.exp(-rate / m);
+        return m - f1 - f2 - (f1 * (tailWeight + m * seen)) / (tailPages + rate * seen);
+    };
+    let low = f1 + f2;
+    let high = f1 + f2 + most;
+    if (g(high) <= 0)
+        return most;
+    for (let step = 0; step < 200 && high - low > 1e-9 * high; step += 1) {
+        const middle = (low + high) / 2;
+        if (g(middle) > 0)
+            high = middle;
+        else
+            low = middle;
+    }
+    return (low + high) / 2 - f1 - f2;
+}
+/** Shlosser's unseen values. */
+function shlosserUnseen(fingerprint, q) {
+    const f1 = valuesOn(fingerprint, 1);
+    return (f1 * tally(fingerprint, pages => (1 - q) ** pages))
+        / tally(fingerprint, pages => pages * q * (1 - q) ** (pages - 1));
+}
+/**
+ * Estimated distinct non-null values in the whole table from a page sample
+ * holding a fraction `q` of its rows (the sampled rows over the table's row
+ * count, so the estimate scales like the row count), never above
+ * `nonNullRows` (the table's estimated non-null rows) or below the values the
+ * sample holds. A `q` of 1 or more (the planner counts no more rows than the
+ * sample holds) leaves no value unseen.
+ */
+export function estimateDistinct(fingerprint, q, nonNullRows) {
+    const d = tally(fingerprint, () => 1);
+    if (d === 0)
+        return 0;
+    if (!(q > 0))
+        throw new Error(`A sample holding values must hold some share of the table's rows (got ${q}).`);
+    const f1 = valuesOn(fingerprint, 1);
+    if (f1 === 0 || q >= 1)
+        return Math.round(d);
+    const f2 = valuesOn(fingerprint, 2);
+    const valuePages = tally(fingerprint, pages => pages);
+    const most = (f1 * (1 - q)) / q;
+    // With no value on exactly two pages the lower bound meets the upper one.
+    const least = f2 === 0 ? most : (f1 * f1) / ((2 * f2 * valuePages) / (valuePages - 1) + (f1 * q) / (1 - q));
+    const unseen = Math.sqrt(adaptiveUnseen(fingerprint, most) * shlosserUnseen(fingerprint, q));
+    return Math.round(Math.max(d, Math.min(nonNullRows, d + Math.min(most, Math.max(least, unseen)))));
+}

package/dist/commands/db-profile-owners.js

@@ -0,0 +1,94 @@
+/**
+ * Who owns a row. With `--owner-table`, a row's owners are the owner-table
+ * rows reached by following foreign keys from the row (child to parent), e.g.
+ * `order_items -> orders -> users`. Every shortest path is used; when a table
+ * reaches the owner table by more than one path (a `buyer_id` and a
+ * `seller_id`), a value must meet the owner threshold along every path.
+ *
+ * Shared by the profiler (which builds SQL from the paths) and `profile show`
+ * (which rebuilds the same paths from the foreign keys in the file, so the
+ * review explains how owners were counted).
+ */
+/** Longer chains are unusual and each hop is a join in every value query. */
+export const OWNER_PATH_MAX_HOPS = 4;
+/** More shortest paths than this points at a modelling problem worth a human look. */
+export const OWNER_PATH_MAX_PATHS = 8;
+export function sameTable(left, right) {
+    return left.schema === right.schema && left.table === right.table;
+}
+export function tableKey(ref) {
+    return `${ref.schema}.${ref.table}`;
+}
+export function ownerReach(table, owner, edges) {
+    if (owner === null)
+        return { kind: 'rows' };
+    if (sameTable(table, owner))
+        return { kind: 'self' };
+    // Breadth-first over child -> parent edges, remembering every edge that
+    // reaches a table at its shortest distance, then enumerate the paths.
+    const outgoing = new Map();
+    for (const edge of edges) {
+        if (sameTable(edge.from, edge.to))
+            continue;
+        const key = tableKey(edge.from);
+        outgoing.set(key, [...(outgoing.get(key) ?? []), edge]);
+    }
+    const distance = new Map([[tableKey(table), 0]]);
+    const arrivals = new Map();
+    let frontier = [table];
+    for (let hop = 1; hop <= OWNER_PATH_MAX_HOPS && frontier.length > 0 && !distance.has(tableKey(owner)); hop += 1) {
+        const next = [];
+        for (const current of frontier) {
+            for (const edge of outgoing.get(tableKey(current)) ?? []) {
+                const target = tableKey(edge.to);
+                const known = distance.get(target);
+                if (known === undefined) {
+                    distance.set(target, hop);
+                    arrivals.set(target, [edge]);
+                    next.push(edge.to);
+                }
+                else if (known === hop) {
+                    arrivals.get(target)?.push(edge);
+                }
+            }
+        }
+        frontier = next;
+    }
+    if (!distance.has(tableKey(owner)))
+        return { kind: 'unreachable' };
+    const paths = [];
+    const walk = (at, suffix) => {
+        if (sameTable(at, table)) {
+            paths.push(suffix);
+            return;
+        }
+        for (const edge of arrivals.get(tableKey(at)) ?? [])
+            walk(edge.from, [edge, ...suffix]);
+    };
+    walk(owner, []);
+    if (paths.length > OWNER_PATH_MAX_PATHS) {
+        throw new Error(`${tableKey(table)} reaches the owner table ${tableKey(owner)} by ${paths.length} different foreign-key paths `
+            + `(more than ${OWNER_PATH_MAX_PATHS}); profile it separately with --schema, or pick an owner table it reaches more directly.`);
+    }
+    paths.sort((left, right) => describePath(left).localeCompare(describePath(right)));
+    return { kind: 'paths', paths };
+}
+export function describePath(path) {
+    return path.map(edge => `${edge.from.table}.${edge.columns.join('+')} -> ${tableKey(edge.to)}`).join(', then ');
+}
+export function describeReach(reach, owner) {
+    switch (reach.kind) {
+        case 'rows':
+            return 'each row counts as one owner (no owner table was named)';
+        case 'self':
+            return 'this is the owner table: each row is one owner';
+        case 'unreachable':
+            return `no foreign-key path (at most ${OWNER_PATH_MAX_HOPS} hops) reaches ${owner === null ? 'the owner table' : tableKey(owner)}, `
+                + 'so owners cannot be counted and no value, shape, JSON key, percentile or share of true from this table is kept';
+        case 'paths':
+            return reach.paths.length === 1
+                ? `owners reached through ${describePath(reach.paths[0])}`
+                : `owners reached through ${reach.paths.length} paths, and a value must pass on every one: `
+                    + reach.paths.map(path => `(${describePath(path)})`).join('; ');
+    }
+}