@haystackeditor/cli 0.17.0 → 0.18.0

package/dist/commands/db-profile.js

@@ -0,0 +1,1020 @@
+/**
+ * `haystack db profile` — a read-only, PII-free profile of a customer's
+ * Postgres database, run by the customer inside their own environment.
+ *
+ * The database computes every number; the profiler never selects a raw row.
+ * The only value strings that reach this process are enum labels and CHECK
+ * value lists (schema, not data) and category values, shapes and JSON key
+ * paths that a `HAVING` clause has already shown to be shared by at least
+ * `categoryMinOwners` owners. Personal columns are decided before any value
+ * query runs, and a personal column never has one. Index definitions also
+ * arrive as written, and every literal in them (a partial-index predicate can
+ * name a value) is replaced with `?` before the definition is kept.
+ *
+ * It runs on a primary or on a read replica (hot standby). A replica keeps no
+ * dead-row counts, so there each table's dead rows are reported unavailable.
+ *
+ * Distinct counts are exact for a table read in full. A table larger than
+ * --scan-max-mb is sampled, but a column whose kind only its distinct count
+ * can settle (category or not) is still counted exactly, over the whole
+ * table, unless the sample alone already holds too many values; a column that
+ * is the whole of a validated foreign key is counted through the parents it
+ * references; every other sampled column is estimated from the sample (see
+ * `db-profile-distinct.ts`). The planner's statistics are never used for them.
+ *
+ * See `db-profile-privacy.ts` for the personal rules, `db-profile-sql.ts` for
+ * every statement, and `db-profile-database.ts` for the read-only guarantees.
+ */
+import { readFileSync, renameSync, writeFileSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import chalk from 'chalk';
+import { DATABASE_PROFILE_VERSION, parseDatabaseProfile, } from './db-profile-contract.js';
+import { ReadOnlyDatabase, runBounded } from './db-profile-database.js';
+import { redactSqlLiterals, SqlLiteralError } from './db-profile-sql-literals.js';
+import { describeReach, ownerReach, tableKey, } from './db-profile-owners.js';
+import { checkConstraintValues, classifyColumnFacts, decideByDistinct, formatShare as pct, jsonPathPersonalShape, keptValueProblem, typeFamily, VALUE_SHAPE_TESTS, } from './db-profile-privacy.js';
+import { estimateDistinct } from './db-profile-distinct.js';
+import { categoryDistinctQuery, categoryValuesQuery, childrenPerParentQuery, hasLengthQuantiles, hasNumberQuantiles, JSON_ROOT_TYPES, jsonKeysQuery, orphanQuery, oversizeQuery, QUANTILE_POINTS, SAMPLE_SEED, sampleDistinctQuery, scanSql, schemaValuesQuery, shapesQuery, statsBatches, statsQuery, } from './db-profile-sql.js';
+/* ------------------------------------------------------------ knobs */
+/** Every knob, its default and its bounds. The README documents what turning
+ * each one up or down does; keep the two in step. */
+export const DB_PROFILE_KNOBS = {
+    categoryMaxDistinct: { flag: '--category-max-distinct', default: 50, min: 1, max: 1_000 },
+    categoryMinOwners: { flag: '--category-min-owners', default: 50, min: 5, max: 1_000_000 },
+    concurrency: { flag: '--concurrency', default: 2, min: 1, max: 8 },
+    statementTimeoutSeconds: { flag: '--statement-timeout', default: 120, min: 1, max: 3_600 },
+    scanMaxMb: { flag: '--scan-max-mb', default: 256, min: 0.01, max: 1_048_576 },
+};
+/** Oldest server this profiler supports (hashtextextended, row_security_active). */
+export const MIN_SERVER_VERSION_NUM = 120_000;
+/** Planner and storage settings only. Never connection, authentication, SSL,
+ * logging or file-location settings. `autovacuum*` and `enable_*` (planner
+ * method switches) are included by prefix. */
+export const SETTINGS_ALLOWLIST = [
+    'shared_buffers', 'work_mem', 'maintenance_work_mem', 'effective_cache_size', 'temp_buffers', 'hash_mem_multiplier',
+    'random_page_cost', 'seq_page_cost', 'cpu_tuple_cost', 'cpu_index_tuple_cost', 'cpu_operator_cost',
+    'parallel_setup_cost', 'parallel_tuple_cost', 'min_parallel_table_scan_size', 'min_parallel_index_scan_size',
+    'effective_io_concurrency', 'maintenance_io_concurrency',
+    'default_statistics_target', 'constraint_exclusion', 'cursor_tuple_fraction', 'from_collapse_limit',
+    'join_collapse_limit', 'geqo', 'geqo_threshold', 'jit', 'jit_above_cost', 'jit_inline_above_cost',
+    'jit_optimize_above_cost', 'plan_cache_mode', 'recursive_worktable_factor',
+    'max_parallel_workers_per_gather', 'max_parallel_workers', 'max_parallel_maintenance_workers', 'max_worker_processes',
+    'vacuum_cost_delay', 'vacuum_cost_limit', 'vacuum_cost_page_hit', 'vacuum_cost_page_miss', 'vacuum_cost_page_dirty',
+    'vacuum_freeze_min_age', 'vacuum_freeze_table_age', 'vacuum_multixact_freeze_min_age',
+    'vacuum_multixact_freeze_table_age', 'default_toast_compression', 'block_size', 'server_encoding',
+    'checkpoint_timeout', 'checkpoint_completion_target', 'max_wal_size', 'min_wal_size', 'wal_buffers',
+];
+export const SETTINGS_PREFIXES = ['autovacuum', 'enable_'];
+const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/u;
+function knob(value, spec, integer) {
+    if (value === undefined)
+        return spec.default;
+    const parsed = Number(value);
+    if (!Number.isFinite(parsed) || (integer && !Number.isInteger(parsed)) || parsed < spec.min || parsed > spec.max) {
+        throw new Error(`${spec.flag} must be ${integer ? 'an integer' : 'a number'} from ${spec.min} to ${spec.max} (got "${value}").`);
+    }
+    return parsed;
+}
+export function parseTableRef(value, flag) {
+    const match = value.match(/^([^.\s]+)\.([^.\s]+)$/u);
+    if (!match)
+        throw new Error(`${flag} must be schema.table, e.g. public.users (got "${value}").`);
+    return { schema: match[1], table: match[2] };
+}
+export function parseDatabaseProfileSettings(options) {
+    if (options.urlEnv === undefined || options.urlEnv.length === 0) {
+        throw new Error('--url-env is required: name the environment variable that holds the connection string, e.g. --url-env DATABASE_URL.');
+    }
+    if (!ENV_NAME.test(options.urlEnv)) {
+        // Never echo the value: it may be a connection string with a password.
+        throw new Error('--url-env takes the NAME of an environment variable (letters, digits, underscore), not a connection string.');
+    }
+    if (options.out === undefined || options.out.length === 0) {
+        throw new Error('--out is required: the file to write the profile to, e.g. --out profile.json.');
+    }
+    const schemas = options.schema === undefined || options.schema.length === 0 ? null : [...new Set(options.schema)];
+    for (const schema of schemas ?? []) {
+        if (schema.trim().length === 0)
+            throw new Error('--schema must not be empty.');
+    }
+    return {
+        urlEnv: options.urlEnv,
+        out: options.out,
+        ownerTable: options.ownerTable === undefined ? null : parseTableRef(options.ownerTable, '--owner-table'),
+        schemas,
+        categoryMaxDistinct: knob(options.categoryMaxDistinct, DB_PROFILE_KNOBS.categoryMaxDistinct, true),
+        categoryMinOwners: knob(options.categoryMinOwners, DB_PROFILE_KNOBS.categoryMinOwners, true),
+        concurrency: knob(options.concurrency, DB_PROFILE_KNOBS.concurrency, true),
+        statementTimeoutMs: knob(options.statementTimeout, DB_PROFILE_KNOBS.statementTimeoutSeconds, true) * 1000,
+        scanMaxBytes: Math.round(knob(options.scanMaxMb, DB_PROFILE_KNOBS.scanMaxMb, false) * 1024 * 1024),
+    };
+}
+export function readConnectionString(name, env = process.env) {
+    const value = env[name];
+    if (value === undefined || value.trim().length === 0) {
+        throw new Error(`Environment variable ${name} is not set. Export the connection string in it (the profiler never takes it as a flag, so it stays out of shell history and process lists).`);
+    }
+    if (!/^postgres(ql)?:\/\//u.test(value.trim())) {
+        throw new Error(`Environment variable ${name} must hold a postgres:// or postgresql:// connection URL.`);
+    }
+    return value.trim();
+}
+/* ------------------------------------------------------------ version */
+export function profilerVersion() {
+    const packagePath = resolve(dirname(fileURLToPath(import.meta.url)), '../../package.json');
+    const manifest = JSON.parse(readFileSync(packagePath, 'utf8'));
+    if (typeof manifest.version !== 'string' || manifest.version.length === 0) {
+        throw new Error(`Could not read the CLI version from ${packagePath}.`);
+    }
+    return `haystack-cli ${manifest.version}`;
+}
+function num(row, key, label) {
+    const value = row[key];
+    const parsed = typeof value === 'number' ? value : typeof value === 'string' ? Number(value) : Number.NaN;
+    if (!Number.isFinite(parsed))
+        throw new Error(`${label}: the database returned no number for ${key}.`);
+    return parsed;
+}
+function numOrNull(row, key, label) {
+    return row[key] === null || row[key] === undefined ? null : num(row, key, label);
+}
+function str(row, key, label) {
+    const value = row[key];
+    if (typeof value !== 'string')
+        throw new Error(`${label}: the database returned no text for ${key}.`);
+    return value;
+}
+function strList(row, key, label) {
+    const value = row[key];
+    if (!Array.isArray(value) || value.some(item => typeof item !== 'string')) {
+        throw new Error(`${label}: the database returned no text list for ${key}.`);
+    }
+    return value;
+}
+function quantiles(row, key, label) {
+    const value = row[key];
+    if (value === null || value === undefined)
+        return null;
+    if (!Array.isArray(value) || value.length !== QUANTILE_POINTS.length) {
+        throw new Error(`${label}: the database returned malformed quantiles for ${key}.`);
+    }
+    const numbers = value.map(item => (typeof item === 'number' ? item : Number(item)));
+    if (numbers.some(item => !Number.isFinite(item)))
+        throw new Error(`${label}: the database returned non-finite quantiles for ${key}.`);
+    const [p5, p25, p50, p75, p95] = numbers.map(item => round(item, 4));
+    return { p5, p25, p50, p75, p95 };
+}
+function round(value, digits) {
+    const factor = 10 ** digits;
+    return Math.round(value * factor) / factor;
+}
+function share(part, whole) {
+    return whole === 0 ? 0 : round(part / whole, 6);
+}
+function rowsWord(count) {
+    return count === 1 ? '1 row' : `${count} rows`;
+}
+function rowsDo(count, where, one, many) {
+    return `${rowsWord(count)}${where} ${count === 1 ? one : many}`;
+}
+function formatBytes(bytes) {
+    const units = ['B', 'kB', 'MB', 'GB', 'TB'];
+    let value = bytes;
+    let unit = 0;
+    while (value >= 1024 && unit < units.length - 1) {
+        value /= 1024;
+        unit += 1;
+    }
+    return `${unit === 0 ? value.toFixed(0) : value.toFixed(1)} ${units[unit]}`;
+}
+async function loadServer(db) {
+    const [server] = await db.query('server version', `SELECT current_setting('server_version') AS version, current_setting('server_version_num') AS num, pg_is_in_recovery() AS in_recovery`);
+    const versionNum = Number(str(server, 'num', 'server version'));
+    if (!Number.isInteger(versionNum) || versionNum < MIN_SERVER_VERSION_NUM) {
+        throw new Error(`Postgres ${str(server, 'version', 'server version')} is older than the oldest supported server (12).`);
+    }
+    if (typeof server.in_recovery !== 'boolean')
+        throw new Error('server version: the database returned no true or false for pg_is_in_recovery().');
+    return { version: str(server, 'version', 'server version'), inRecovery: server.in_recovery };
+}
+async function loadSettings(db) {
+    const prefixes = SETTINGS_PREFIXES.map(prefix => `name LIKE '${prefix.replace('_', '\\_')}%'`).join(' OR ');
+    const rows = await db.query('allowlisted settings', `SELECT name, current_setting(name) AS value FROM pg_settings WHERE name = ANY($1::text[]) OR ${prefixes} ORDER BY name`, [SETTINGS_ALLOWLIST]);
+    return rows.map(row => ({ name: str(row, 'name', 'settings'), value: str(row, 'value', 'settings') }));
+}
+async function loadSchemas(db, requested) {
+    if (requested === null) {
+        const rows = await db.query('schemas', `SELECT nspname::text AS name FROM pg_namespace WHERE nspname <> 'information_schema' AND nspname !~ '^pg_' ORDER BY 1`);
+        return rows.map(row => str(row, 'name', 'schemas'));
+    }
+    const rows = await db.query('schemas', `SELECT nspname::text AS name FROM pg_namespace WHERE nspname = ANY($1::text[])`, [requested]);
+    const found = new Set(rows.map(row => str(row, 'name', 'schemas')));
+    const missing = requested.filter(schema => !found.has(schema));
+    if (missing.length > 0)
+        throw new Error(`--schema names schemas that do not exist: ${missing.join(', ')}.`);
+    return requested;
+}
+async function loadTables(db, schemas) {
+    const rows = await db.query('tables', `SELECT c.oid::text AS oid, n.nspname::text AS schema, c.relname::text AS name,
+  c.relkind::text AS relkind, c.relispartition AS partition, c.reltuples::float8 AS reltuples,
+  pg_relation_size(c.oid)::float8 AS heap_bytes, pg_table_size(c.oid)::float8 AS table_size,
+  pg_indexes_size(c.oid)::float8 AS index_bytes,
+  (CASE WHEN c.reltoastrelid = 0 THEN 0 ELSE pg_total_relation_size(c.reltoastrelid) END)::float8 AS toast_bytes,
+  has_table_privilege(c.oid, 'SELECT') AS can_select, row_security_active(c.oid) AS rls_active,
+  s.n_live_tup::float8 AS live, s.n_dead_tup::float8 AS dead,
+  EXISTS (SELECT 1 FROM pg_depend d WHERE d.classid = 'pg_class'::regclass AND d.objid = c.oid AND d.deptype = 'e') AS extension_member
+FROM pg_class c
+JOIN pg_namespace n ON n.oid = c.relnamespace
+LEFT JOIN pg_stat_user_tables s ON s.relid = c.oid
+WHERE n.nspname = ANY($1::text[]) AND c.relkind IN ('r', 'p')
+ORDER BY n.nspname, c.relname`, [schemas]);
+    return rows.map(row => ({
+        oid: str(row, 'oid', 'tables'),
+        ref: { schema: str(row, 'schema', 'tables'), table: str(row, 'name', 'tables') },
+        relkind: str(row, 'relkind', 'tables'),
+        partition: row.partition === true,
+        reltuples: num(row, 'reltuples', 'tables'),
+        heapBytes: num(row, 'heap_bytes', 'tables'),
+        tableSize: num(row, 'table_size', 'tables'),
+        indexBytes: num(row, 'index_bytes', 'tables'),
+        toastBytes: num(row, 'toast_bytes', 'tables'),
+        canSelect: row.can_select === true,
+        rlsActive: row.rls_active === true,
+        live: numOrNull(row, 'live', 'tables'),
+        dead: numOrNull(row, 'dead', 'tables'),
+        extensionMember: row.extension_member === true,
+    }));
+}
+async function loadColumns(db, oids) {
+    const rows = await db.query('columns', `SELECT a.attrelid::text AS table_oid, a.attname::text AS name,
+  format_type(a.atttypid, a.atttypmod) AS data_type, a.attnotnull AS not_null,
+  b.oid::text AS base_oid, b.typname::text AS typname, b.typcategory::text AS typcategory, b.typtype::text AS typtype,
+  EXISTS (SELECT 1 FROM pg_opclass oc JOIN pg_am am ON am.oid = oc.opcmethod
+          WHERE am.amname = 'btree' AND oc.opcdefault AND oc.opcintype = b.oid) AS sortable
+FROM pg_attribute a
+JOIN pg_type t ON t.oid = a.atttypid
+JOIN pg_type b ON b.oid = (CASE WHEN t.typtype = 'd' THEN t.typbasetype ELSE t.oid END)
+WHERE a.attrelid = ANY($1::text[]::oid[]) AND a.attnum > 0 AND NOT a.attisdropped
+ORDER BY a.attrelid, a.attnum`, [oids]);
+    return rows.map(row => ({
+        tableOid: str(row, 'table_oid', 'columns'),
+        name: str(row, 'name', 'columns'),
+        dataType: str(row, 'data_type', 'columns'),
+        notNull: row.not_null === true,
+        baseOid: str(row, 'base_oid', 'columns'),
+        typname: str(row, 'typname', 'columns'),
+        family: typeFamily({
+            typname: str(row, 'typname', 'columns'),
+            typcategory: str(row, 'typcategory', 'columns'),
+            typtype: str(row, 'typtype', 'columns'),
+        }),
+        sortable: row.sortable === true,
+    }));
+}
+async function loadConstraints(db, oids) {
+    const rows = await db.query('constraints', `SELECT c.conrelid::text AS table_oid, c.conname::text AS name, c.contype::text AS type,
+  c.convalidated AS validated,
+  ARRAY(SELECT a.attname::text FROM unnest(c.conkey) WITH ORDINALITY k(attnum, ord)
+        JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = k.attnum ORDER BY k.ord) AS columns,
+  rn.nspname::text AS ref_schema, rc.relname::text AS ref_table,
+  ARRAY(SELECT a.attname::text FROM unnest(c.confkey) WITH ORDINALITY k(attnum, ord)
+        JOIN pg_attribute a ON a.attrelid = c.confrelid AND a.attnum = k.attnum ORDER BY k.ord) AS ref_columns,
+  CASE WHEN c.contype = 'c' AND array_length(c.conkey, 1) = 1 THEN pg_get_constraintdef(c.oid) END AS check_definition
+FROM pg_constraint c
+LEFT JOIN pg_class rc ON rc.oid = c.confrelid
+LEFT JOIN pg_namespace rn ON rn.oid = rc.relnamespace
+WHERE c.conrelid = ANY($1::text[]::oid[]) AND c.contype IN ('p', 'u', 'f', 'c')
+ORDER BY c.conrelid, c.conname`, [oids]);
+    return rows.map(row => {
+        const type = str(row, 'type', 'constraints');
+        return {
+            tableOid: str(row, 'table_oid', 'constraints'),
+            name: str(row, 'name', 'constraints'),
+            type,
+            validated: row.validated === true,
+            columns: strList(row, 'columns', 'constraints'),
+            referenced: type === 'f'
+                ? { schema: str(row, 'ref_schema', 'constraints'), table: str(row, 'ref_table', 'constraints') }
+                : null,
+            referencedColumns: type === 'f' ? strList(row, 'ref_columns', 'constraints') : [],
+            checkDefinition: typeof row.check_definition === 'string' ? row.check_definition : null,
+        };
+    });
+}
+async function loadEnumLabels(db, typeOids) {
+    if (typeOids.length === 0)
+        return new Map();
+    const rows = await db.query('enum labels', `SELECT enumtypid::text AS type_oid, array_agg(enumlabel::text ORDER BY enumsortorder) AS labels
+FROM pg_enum WHERE enumtypid = ANY($1::text[]::oid[]) GROUP BY enumtypid`, [typeOids]);
+    return new Map(rows.map(row => [str(row, 'type_oid', 'enum labels'), strList(row, 'labels', 'enum labels')]));
+}
+async function loadIndexes(db, schemas) {
+    const rows = await db.query('indexes', `SELECT schemaname::text AS schema, tablename::text AS table_name, indexname::text AS name, indexdef AS definition
+FROM pg_indexes WHERE schemaname = ANY($1::text[]) ORDER BY schemaname, tablename, indexname`, [schemas]);
+    const byTable = new Map();
+    for (const row of rows) {
+        const key = tableKey({ schema: str(row, 'schema', 'indexes'), table: str(row, 'table_name', 'indexes') });
+        const name = str(row, 'name', 'indexes');
+        let definition;
+        try {
+            definition = redactSqlLiterals(str(row, 'definition', 'indexes'));
+        }
+        catch (error) {
+            // Never quote the definition: it is exactly what may hold a value.
+            if (error instanceof SqlLiteralError) {
+                throw new Error(`Index ${str(row, 'schema', 'indexes')}.${name}: its definition cannot be checked for literal values (${error.message}), so the profile cannot include it; report this to Haystack.`);
+            }
+            throw error;
+        }
+        byTable.set(key, [...(byTable.get(key) ?? []), { name, definition }]);
+    }
+    return byTable;
+}
+/** Rounded to the six decimals the TABLESAMPLE clause and the profile carry. */
+function samplePercentFor(heapBytes, scanMaxBytes) {
+    if (heapBytes <= scanMaxBytes)
+        return null;
+    return round(Math.max(0.000001, Math.min(99.999999, (100 * scanMaxBytes) / heapBytes)), 6);
+}
+/* ------------------------------------------------------------ profile */
+export async function profileDatabase(db, run, log, now = () => new Date()) {
+    const server = await loadServer(db);
+    if (server.inRecovery) {
+        log('Connected to a read replica (hot standby): dead-row shares are reported as unavailable, because Postgres counts dead rows only on the primary.');
+    }
+    const settings = await loadSettings(db);
+    const schemas = await loadSchemas(db, run.schemas);
+    const allTables = await loadTables(db, schemas);
+    const partitioned = allTables.filter(table => table.relkind === 'p');
+    if (partitioned.length > 0) {
+        log(`Partitioned tables are profiled through their partitions, not as parents: ${partitioned.map(table => tableKey(table.ref)).join(', ')}.`);
+    }
+    const extensionTables = allTables.filter(table => table.relkind === 'r' && table.extensionMember);
+    if (extensionTables.length > 0) {
+        log(`Skipping tables that belong to installed extensions, not the application: ${extensionTables.map(table => tableKey(table.ref)).join(', ')}.`);
+    }
+    const tables = allTables.filter(table => table.relkind === 'r' && !table.extensionMember);
+    if (tables.length === 0)
+        throw new Error(`No application tables found in schema${schemas.length === 1 ? '' : 's'} ${schemas.join(', ')}.`);
+    const unreadable = tables.filter(table => !table.canSelect);
+    if (unreadable.length > 0) {
+        throw new Error(`The profiler's role cannot SELECT from ${unreadable.map(table => tableKey(table.ref)).join(', ')}; grant SELECT (read only) or narrow --schema.`);
+    }
+    const rowSecured = tables.filter(table => table.rlsActive);
+    if (rowSecured.length > 0) {
+        throw new Error(`Row-level security hides rows of ${rowSecured.map(table => tableKey(table.ref)).join(', ')} from the profiler's role, so the profile would be partial; use a role with BYPASSRLS or narrow --schema.`);
+    }
+    if (run.ownerTable !== null && !tables.some(table => tableKey(table.ref) === tableKey(run.ownerTable))) {
+        throw new Error(`--owner-table ${tableKey(run.ownerTable)} is not an application table in the profiled schemas (${schemas.join(', ')}).`);
+    }
+    const oids = tables.map(table => table.oid);
+    const columns = await loadColumns(db, oids);
+    const constraints = await loadConstraints(db, oids);
+    const enumLabels = await loadEnumLabels(db, [...new Set(columns.filter(column => column.family === 'enum').map(column => column.baseOid))]);
+    const indexes = await loadIndexes(db, schemas);
+    const tableByOid = new Map(tables.map(table => [table.oid, table]));
+    const edges = [];
+    const foreignKeysByTable = new Map();
+    for (const constraint of constraints) {
+        if (constraint.type !== 'f' || constraint.referenced === null)
+            continue;
+        const table = tableByOid.get(constraint.tableOid);
+        if (table === undefined)
+            continue;
+        const edge = {
+            name: constraint.name,
+            from: table.ref,
+            columns: constraint.columns,
+            to: constraint.referenced,
+            referencedColumns: constraint.referencedColumns,
+        };
+        edges.push(edge);
+        foreignKeysByTable.set(table.oid, [...(foreignKeysByTable.get(table.oid) ?? []), { edge, validated: constraint.validated }]);
+    }
+    // Owner paths may only run through profiled tables: `profile show` rebuilds
+    // them from the file, which holds only profiled tables' foreign keys.
+    const profiled = new Set(tables.map(table => tableKey(table.ref)));
+    const ownerEdges = edges.filter(edge => profiled.has(tableKey(edge.to)));
+    log(`Profiling ${tables.length} table${tables.length === 1 ? '' : 's'} in ${schemas.join(', ')} with ${db.concurrency} connection${db.concurrency === 1 ? '' : 's'}.`);
+    const plans = tables.map(table => {
+        const label = tableKey(table.ref);
+        const reach = ownerReach(table.ref, run.ownerTable, ownerEdges);
+        const samplePercent = samplePercentFor(table.heapBytes, run.scanMaxBytes);
+        const tableConstraints = constraints.filter(constraint => constraint.tableOid === table.oid);
+        const keyColumns = new Set(tableConstraints
+            .filter(constraint => constraint.type === 'p' || constraint.type === 'f' || (constraint.type === 'u' && constraint.columns.length === 1))
+            .flatMap(constraint => constraint.columns));
+        const planColumns = columns.filter(column => column.tableOid === table.oid).map(column => {
+            let schemaValues = null;
+            if (column.family === 'enum') {
+                const labels = enumLabels.get(column.baseOid);
+                if (labels === undefined)
+                    throw new Error(`${label}.${column.name}: enum type ${column.dataType} has no labels in pg_enum.`);
+                schemaValues = labels;
+            }
+            else {
+                for (const constraint of tableConstraints) {
+                    if (constraint.type !== 'c' || constraint.checkDefinition === null || constraint.columns[0] !== column.name)
+                        continue;
+                    const values = checkConstraintValues(constraint.checkDefinition, column.name);
+                    if (values !== null) {
+                        schemaValues = values;
+                        break;
+                    }
+                }
+            }
+            return {
+                name: column.name,
+                family: column.family,
+                typname: column.typname,
+                sortable: column.sortable,
+                catalog: column,
+                schemaValues,
+                isKey: keyColumns.has(column.name),
+            };
+        });
+        log(`  ${label}: ${samplePercent === null
+            ? `reading all ${formatBytes(table.heapBytes)}`
+            : `reading a ${pct(samplePercent / 100)} TABLESAMPLE SYSTEM sample (about ${formatBytes(run.scanMaxBytes)} of ${formatBytes(table.heapBytes)})`}; ${describeReach(reach, run.ownerTable)}.`);
+        const foreignKeys = foreignKeysByTable.get(table.oid) ?? [];
+        const uniqueColumns = new Set();
+        const keyCoverage = new Map();
+        if (samplePercent !== null) {
+            for (const constraint of tableConstraints) {
+                if ((constraint.type === 'p' || constraint.type === 'u') && constraint.columns.length === 1)
+                    uniqueColumns.add(constraint.columns[0]);
+            }
+            for (const { edge, validated } of foreignKeys) {
+                const [only] = edge.columns;
+                if (validated && edge.columns.length === 1 && !uniqueColumns.has(only) && !keyCoverage.has(only))
+                    keyCoverage.set(only, edge);
+            }
+        }
+        return {
+            table,
+            label,
+            columns: planColumns,
+            reach,
+            samplePercent,
+            scan: scanSql({ table: table.ref, samplePercent, reach }),
+            foreignKeys,
+            uniqueColumns,
+            keyCoverage,
+            indexes: indexes.get(label) ?? [],
+        };
+    });
+    // Pass 1: one aggregate scan per table (more for very wide tables).
+    const stats = new Map();
+    const statsWork = plans.flatMap(plan => {
+        const batches = statsBatches(plan.columns, plan.samplePercent === null);
+        return batches.map((batch, batchIndex) => ({ plan, batch, part: batches.length === 1 ? '' : ` (part ${batchIndex + 1} of ${batches.length})` }));
+    });
+    await runBounded(statsWork, db.concurrency, async ({ plan, batch, part }) => {
+        const [row] = await db.query(`${plan.label}: column statistics${part}`, statsQuery(plan.scan, batch, plan.samplePercent === null));
+        stats.set(plan, { ...(stats.get(plan) ?? {}), ...row });
+    });
+    // Classify every column before any value query exists. A table read in
+    // full is settled here from its exact distinct counts; a sampled table's
+    // columns that only a distinct count can settle wait for the passes below.
+    const columnStates = new Map();
+    for (const plan of plans) {
+        columnStates.set(plan, plan.columns.map((column, index) => columnState(plan, column, index, stats.get(plan), run)));
+    }
+    // Pass 1b: each sampled column's distinct values and the sampled pages each
+    // is found on, for its estimate, except key columns counted another way.
+    const sampleWork = plans.filter(plan => plan.samplePercent !== null).flatMap(plan => columnStates.get(plan)
+        .filter(state => !plan.uniqueColumns.has(state.plan.name) && !plan.keyCoverage.has(state.plan.name))
+        .map(state => ({ plan, state })));
+    await runBounded(sampleWork, db.concurrency, async ({ plan, state }) => {
+        const at = `${plan.label}.${state.plan.name}`;
+        const rows = await db.query(`${at}: distinct values in the sample`, sampleDistinctQuery(plan.table.ref, plan.samplePercent, state.plan));
+        state.sample = rows.map(row => ({ pages: num(row, 'pages', at), values: num(row, 'value_count', at) }));
+    });
+    // Pass 1c: a category column rests on an exact count over the whole table,
+    // never on an estimate. A sample never holds more distinct values than its
+    // table, so one with more than categoryMaxDistinct already rules the column
+    // out; the rest are counted exactly, all of a table's in one read of it.
+    const categoryWork = [];
+    for (const plan of plans) {
+        if (plan.samplePercent === null)
+            continue;
+        const candidates = [];
+        for (const state of columnStates.get(plan)) {
+            if (state.verdict.kind !== 'distinct-decides')
+                continue;
+            const inSample = sampleOf(state, plan).reduce((sum, entry) => sum + entry.values, 0);
+            if (inSample > run.categoryMaxDistinct)
+                settle(state, decideByDistinct(state.verdict, inSample, run.categoryMaxDistinct));
+            else
+                candidates.push(state);
+        }
+        if (candidates.length === 0)
+            continue;
+        const names = candidates.map(state => state.plan.name);
+        log(`  ${plan.label}: reading the whole table once to count the distinct values of ${names.join(', ')} exactly, since ${names.length === 1 ? 'that decides whether it is a category column' : 'those decide whether each is a category column'}.`);
+        categoryWork.push({ plan, candidates });
+    }
+    await runBounded(categoryWork, db.concurrency, async ({ plan, candidates }) => {
+        const [row] = await db.query(`${plan.label}: exact distinct counts of possible category columns (reads the whole table)`, categoryDistinctQuery(plan.table.ref, candidates.map(state => ({ index: state.index, column: state.plan }))));
+        for (const state of candidates) {
+            const exact = num(row, `c${state.index}_d`, `${plan.label}.${state.plan.name}`);
+            state.distinctEstimate = exact;
+            settle(state, decideByDistinct(state.verdict, exact, run.categoryMaxDistinct));
+        }
+    });
+    // Pass 2: value, shape, JSON key, oversize and foreign-key queries.
+    const tasks = [];
+    const foreignKeyProfiles = new Map();
+    const tableByKey = new Map(tables.map(table => [tableKey(table.ref), table]));
+    const planByKey = new Map(plans.map(plan => [plan.label, plan]));
+    for (const plan of plans) {
+        const states = columnStates.get(plan);
+        const ownersCountable = plan.reach.kind !== 'unreachable';
+        for (const state of states) {
+            const column = state.plan;
+            if (state.nonNull === 0) {
+                // Schema-declared values need no data: an empty column still lists them.
+                if (settled(state).kind === 'category' && column.schemaValues !== null) {
+                    state.categories = column.schemaValues.map(value => ({ value, fraction: 0, source: 'schema' }));
+                }
+                continue;
+            }
+            const at = `${plan.label}.${column.name}`;
+            const kind = settled(state).kind;
+            if (kind === 'category' && column.schemaValues !== null) {
+                tasks.push(async () => {
+                    const rows = await db.query(`${at}: schema category counts`, schemaValuesQuery(plan.scan, column.name), [column.schemaValues]);
+                    const counts = new Map(rows.map(row => [str(row, 'value', at), num(row, 'rows', at)]));
+                    state.categories = column.schemaValues.map(value => ({
+                        value,
+                        fraction: share(counts.get(value) ?? 0, state.nonNull),
+                        source: 'schema',
+                    }));
+                });
+            }
+            else if (kind === 'category' && ownersCountable) {
+                tasks.push(async () => {
+                    const rows = await db.query(`${at}: category values`, categoryValuesQuery(plan.scan, column.name), [run.categoryMinOwners]);
+                    const values = rows.map(row => ({ value: str(row, 'value', at), rows: num(row, 'rows', at) }));
+                    // A value of a personal shape, or one that does not fit the
+                    // column's declared type, never leaves however many owners share it
+                    // (keptValueProblem, the rule the upload server applies too): it is
+                    // dropped exactly like a value that fails the owner rule (left out,
+                    // its rows counted as withheld, the other values' shares unchanged).
+                    // The value itself is never logged.
+                    const problems = values.map(entry => keptValueProblem(column.catalog.dataType, entry.value));
+                    const dropped = problems.filter((problem) => problem !== null);
+                    if (dropped.length > 0) {
+                        log(`  ${at}: not keeping ${dropped.length === 1 ? 'a value that' : `${dropped.length} values that each`} ${[...new Set(dropped)].join('; or ')}, though shared by enough owners.`);
+                    }
+                    state.categories = values
+                        .filter((_, index) => problems[index] === null)
+                        .map(entry => ({ value: entry.value, fraction: share(entry.rows, state.nonNull), source: 'data' }));
+                });
+            }
+            if (column.family === 'text' && kind !== 'free-text' && ownersCountable) {
+                tasks.push(async () => {
+                    const rows = await db.query(`${at}: shapes`, shapesQuery(plan.scan, column.name), [run.categoryMinOwners]);
+                    state.shapeRows = rows.map(row => ({ shape: str(row, 'shape', at), rows: num(row, 'rows', at) }));
+                });
+            }
+            if (column.family === 'json' && ownersCountable) {
+                tasks.push(async () => {
+                    const rows = await db.query(`${at}: JSON key paths`, jsonKeysQuery(plan.scan, column.name), [run.categoryMinOwners]);
+                    const entries = rows.map(row => ({
+                        path: str(row, 'path', at),
+                        present: num(row, 'present', at),
+                        type: str(row, 'type', at),
+                        rows: num(row, 'rows', at),
+                    }));
+                    // The query already left out paths with a personal key or too few
+                    // owners. A path whose keys only read as personal together (run by
+                    // run, as the upload check tests it) is withheld here, entirely: no
+                    // key entry and no oddity is built for it, since both come from
+                    // these rows alone. The path itself is never logged.
+                    const shaped = [...new Set(entries.map(entry => entry.path))]
+                        .map(path => jsonPathPersonalShape(path))
+                        .filter((what) => what !== null);
+                    if (shaped.length > 0) {
+                        log(`  ${at}: not keeping ${shaped.length === 1 ? 'a JSON key path' : `${shaped.length} JSON key paths`} shaped like ${[...new Set(shaped)].join(', ')}, though shared by enough owners.`);
+                    }
+                    state.jsonRows = entries.filter(entry => jsonPathPersonalShape(entry.path) === null);
+                });
+            }
+        }
+        const oversizeChecks = states
+            .map((state, index) => ({ state, index }))
+            .filter(({ state }) => hasLengthQuantiles(state.plan.family) && state.lengthP95 !== null)
+            .map(({ state, index }) => ({
+            index,
+            column: state.plan,
+            thresholdBytes: Math.max(10 * state.lengthP95, 1024),
+            state,
+        }));
+        if (oversizeChecks.length > 0) {
+            tasks.push(async () => {
+                const [row] = await db.query(`${plan.label}: oversize values`, oversizeQuery(plan.scan, oversizeChecks));
+                for (const check of oversizeChecks)
+                    check.state.oversize = num(row, `c${check.index}_big`, plan.label);
+            });
+        }
+        const keys = [];
+        foreignKeyProfiles.set(plan, keys);
+        for (const { edge, validated } of plan.foreignKeys) {
+            const at = `${plan.label} foreign key ${edge.name}`;
+            const profileEntry = {
+                name: edge.name,
+                columns: edge.columns,
+                references: { schema: edge.to.schema, table: edge.to.table, columns: edge.referencedColumns },
+                childrenPerParent: { p5: 0, p25: 0, p50: 0, p75: 0, p95: 0 },
+                orphanFraction: 0,
+            };
+            keys.push(profileEntry);
+            const parent = tableByKey.get(tableKey(edge.to));
+            const parentSample = parent === undefined ? null : samplePercentFor(parent.heapBytes, run.scanMaxBytes);
+            const covered = plan.keyCoverage.get(edge.columns[0]) === edge
+                ? states.find(state => state.plan.name === edge.columns[0])
+                : undefined;
+            tasks.push(async () => {
+                const [row] = await db.query(`${at}: children per parent`, childrenPerParentQuery(edge, parentSample));
+                const measured = quantiles(row, 'q', at);
+                // An empty parent table has no children-per-parent distribution; the
+                // profile records zeros, and the parent's row count of 0 says why.
+                if (measured !== null)
+                    profileEntry.childrenPerParent = measured;
+                if (covered !== undefined) {
+                    // Every child value matches exactly one parent key, so the parents
+                    // with a child are the column's distinct values.
+                    const referenced = num(row, 'referenced', at);
+                    covered.distinctEstimate = parentSample === null
+                        ? referenced
+                        : referencedInTable(referenced, planByKey.get(tableKey(edge.to)), stats, `${plan.label}.${edge.columns[0]}`);
+                }
+            });
+            if (!validated) {
+                tasks.push(async () => {
+                    const [row] = await db.query(`${at}: orphans`, orphanQuery(edge, plan.samplePercent));
+                    profileEntry.orphanFraction = share(num(row, 'orphans', at), num(row, 'children', at));
+                });
+            }
+        }
+    }
+    await runBounded(tasks, db.concurrency, task => task());
+    const tableProfiles = plans.map(plan => {
+        const row = stats.get(plan);
+        const total = num(row, 'total', plan.label);
+        const rowCount = plan.samplePercent === null ? total : plannerRowCount(plan, total);
+        for (const state of columnStates.get(plan)) {
+            if (state.distinctEstimate !== null)
+                continue;
+            // Scaled like the row count: the table's rows times the sample's non-null share.
+            const nonNullRows = total === 0 ? 0 : (rowCount * state.nonNull) / total;
+            state.distinctEstimate = plan.uniqueColumns.has(state.plan.name)
+                ? nonNullRows
+                : estimateDistinct(sampleOf(state, plan), total / rowCount, nonNullRows);
+        }
+        return {
+            schema: plan.table.ref.schema,
+            name: plan.table.ref.table,
+            rowCount: Math.round(rowCount),
+            rowCountSource: plan.samplePercent === null ? 'exact' : 'estimate',
+            sampling: plan.samplePercent === null
+                ? { status: 'full-scan' }
+                : { status: 'sampled', percent: plan.samplePercent, seed: SAMPLE_SEED },
+            tableBytes: Math.round(plan.table.tableSize - plan.table.toastBytes),
+            indexBytes: Math.round(plan.table.indexBytes),
+            toastBytes: Math.round(plan.table.toastBytes),
+            deadRows: deadRows(plan, rowCount, server.inRecovery),
+            columns: columnStates.get(plan).map(state => finishColumn(state, plan, run)),
+            foreignKeys: foreignKeyProfiles.get(plan).sort((left, right) => left.name.localeCompare(right.name)),
+            indexes: plan.indexes,
+        };
+    });
+    const rules = {
+        categoryMaxDistinct: run.categoryMaxDistinct,
+        categoryMinOwners: run.categoryMinOwners,
+        ownerCounting: run.ownerTable === null
+            ? {
+                kind: 'rows',
+                reason: `No --owner-table was named, so each row counts as one owner: a value is kept only when at least ${run.categoryMinOwners} rows share it, even if they all belong to one person.`,
+            }
+            : { kind: 'owner-table', schema: run.ownerTable.schema, table: run.ownerTable.table },
+    };
+    const profile = {
+        version: DATABASE_PROFILE_VERSION,
+        profilerVersion: profilerVersion(),
+        createdAt: now().toISOString(),
+        engine: 'postgres',
+        engineVersion: server.version,
+        settings,
+        rules,
+        tables: tableProfiles,
+    };
+    // The file must satisfy the same strict parser `show` and `upload` use.
+    return parseDatabaseProfile(profile);
+}
+/** A sampled parent's referenced rows scaled to its whole table, like its row count. */
+function referencedInTable(referenced, parent, stats, at) {
+    const sampledRows = num(stats.get(parent), 'total', parent.label);
+    if (sampledRows === 0) {
+        throw new Error(`${at}: the ${pct(parent.samplePercent / 100)} sample of ${parent.label} holds no rows, so how many of its rows ${at} references cannot be estimated; VACUUM ${parent.label} (its sampled pages are empty) or raise --scan-max-mb to read it in full.`);
+    }
+    return Math.round((referenced * plannerRowCount(parent, sampledRows)) / sampledRows);
+}
+/**
+ * The planner's row count of a sampled table (`sampledRows` is its sample's
+ * count). Postgres 14 and later mark a never-analyzed table -1; 12 and 13
+ * leave it 0, which is also what an analyzed empty table has. So 0 counts as
+ * unknown when the sample itself holds rows (checking only that the heap has
+ * pages would refuse a table emptied and analyzed however often ANALYZE ran).
+ */
+function plannerRowCount(plan, sampledRows) {
+    const reltuples = plan.table.reltuples;
+    if (reltuples < 0 || (reltuples === 0 && sampledRows > 0)) {
+        throw new Error(`${plan.label} is larger than --scan-max-mb and has never been analyzed${reltuples === 0 ? ' (or not since its rows were written)' : ''}, so its row count is unknown without a full count; run ANALYZE ${plan.label} (or let autovacuum do so), or raise --scan-max-mb to count it in full.`);
+    }
+    return reltuples;
+}
+export const REPLICA_DEAD_ROWS_REASON = 'Profiled on a read replica (hot standby): Postgres counts dead rows only on the primary, so their share cannot be measured here.';
+function deadRows(plan, rowCount, inRecovery) {
+    // A standby's pg_stat_user_tables shows 0 live and 0 dead rows for every
+    // table: the counters are kept by the server that writes, not replayed.
+    if (inRecovery)
+        return { status: 'unavailable', reason: REPLICA_DEAD_ROWS_REASON };
+    const live = plan.table.live ?? 0;
+    const dead = plan.table.dead ?? 0;
+    if (live + dead > 0)
+        return { status: 'measured', fraction: round(dead / (live + dead), 6) };
+    if (rowCount === 0)
+        return { status: 'measured', fraction: 0 };
+    throw new Error(`${plan.label} has rows but pg_stat_user_tables has no live or dead row counts for it (its statistics were reset, or it was never vacuumed or analyzed), so its dead-row share cannot be measured; run ANALYZE ${plan.label} and profile again.`);
+}
+function settle(state, classification) {
+    state.verdict = classification;
+    state.categories = classification.kind === 'category' ? [] : null;
+}
+function settled(state) {
+    if (state.verdict.kind === 'distinct-decides')
+        throw new Error(`${state.plan.name}: classified before its distinct count was known.`);
+    return state.verdict;
+}
+function sampleOf(state, plan) {
+    if (state.sample === null)
+        throw new Error(`${plan.label}.${state.plan.name}: its sample's distinct values were never counted.`);
+    return state.sample;
+}
+function columnState(plan, column, index, row, run) {
+    const key = `c${index}`;
+    const at = `${plan.label}.${column.name}`;
+    const total = num(row, 'total', at);
+    const nonNull = num(row, `${key}_nn`, at);
+    const exactDistinct = plan.samplePercent === null ? num(row, `${key}_nd`, at) : null;
+    const lengthQuantiles = hasLengthQuantiles(column.family) ? quantiles(row, `${key}_lq`, at) : null;
+    const valueShapes = column.family === 'text'
+        ? {
+            nonNull,
+            matches: Object.fromEntries(VALUE_SHAPE_TESTS.map(test => [test.id, num(row, `${key}_t_${test.id.replace('-', '_')}`, at)])),
+            averageWords: numOrNull(row, `${key}_words`, at),
+            fourWordValues: num(row, `${key}_w4`, at),
+        }
+        : null;
+    const facts = classifyColumnFacts({
+        name: column.name,
+        dataType: column.catalog.dataType,
+        family: column.family,
+        isKey: column.isKey,
+        schemaValues: column.schemaValues,
+        valueShapes,
+    });
+    const verdict = facts.kind === 'distinct-decides' && exactDistinct !== null
+        ? decideByDistinct(facts, exactDistinct, run.categoryMaxDistinct)
+        : facts;
+    return {
+        plan: column,
+        index,
+        total,
+        nonNull,
+        verdict,
+        sample: null,
+        distinctEstimate: exactDistinct,
+        avgWidth: nonNull === 0 ? 0 : round(num(row, `${key}_aw`, at), 2),
+        owners: num(row, `${key}_own`, at),
+        lengthQuantiles,
+        lengthP95: lengthQuantiles?.p95 ?? null,
+        numberQuantiles: hasNumberQuantiles(column.family) ? quantiles(row, `${key}_nq`, at) : null,
+        nonFinite: hasNumberQuantiles(column.family) ? num(row, `${key}_nonfinite`, at) : 0,
+        numericText: column.family === 'text' ? num(row, `${key}_numeric`, at) : 0,
+        dateText: column.family === 'text' ? num(row, `${key}_date`, at) : 0,
+        badCharacters: column.family === 'text' ? num(row, `${key}_bad`, at) : 0,
+        jsonRoots: column.family === 'json'
+            ? Object.fromEntries(JSON_ROOT_TYPES.map(type => [type, num(row, `${key}_j_${type}`, at)]))
+            : null,
+        trueRows: column.family === 'boolean' ? num(row, `${key}_true`, at) : null,
+        categories: verdict.kind === 'category' ? [] : null,
+        shapeRows: null,
+        jsonRows: null,
+        oversize: 0,
+    };
+}
+function finishColumn(state, plan, run) {
+    const { kind, personalReason } = settled(state);
+    if (state.distinctEstimate === null)
+        throw new Error(`${plan.label}.${state.plan.name}: its distinct count was never measured.`);
+    // Quantiles over fewer owners than the threshold sit next to a real
+    // person's value (p5 of three rows is nearly the minimum), so they are kept
+    // only when at least `categoryMinOwners` owners have a value here. A table
+    // the owner table is unreachable from has no owner count (its `owners` is a
+    // row count, and 50 rows can be one person's), so nothing owner-gated
+    // leaves it: no percentiles, no share of true, as no value, shape or key.
+    const quantilesAllowed = plan.reach.kind !== 'unreachable' && state.owners >= run.categoryMinOwners;
+    // A percentile of a personal number is itself a personal value (the median
+    // birth date is somebody's birth date; the median of a numeric phone column
+    // is a phone number), so personal columns keep length quantiles only.
+    const personal = kind === 'personal' || kind === 'free-text';
+    // A true share over fewer owners describes those few owners (one user's
+    // flag in a one-row table), so it is withheld on the same rule.
+    const booleanTrueFraction = kind === 'boolean' && state.trueRows !== null && state.nonNull > 0 && quantilesAllowed
+        ? share(state.trueRows, state.nonNull)
+        : null;
+    const shapes = (state.shapeRows ?? []).map(entry => ({ shape: entry.shape, fraction: share(entry.rows, state.nonNull) }));
+    const jsonKeys = jsonKeyProfiles(state);
+    return {
+        name: state.plan.name,
+        dataType: state.plan.catalog.dataType,
+        notNull: state.plan.catalog.notNull,
+        nullFraction: share(state.total - state.nonNull, state.total),
+        distinctEstimate: Math.round(state.distinctEstimate),
+        avgWidthBytes: state.avgWidth,
+        kind,
+        personalReason,
+        categories: state.categories,
+        lengthQuantiles: quantilesAllowed ? state.lengthQuantiles : null,
+        numberQuantiles: quantilesAllowed && !personal ? state.numberQuantiles : null,
+        booleanTrueFraction,
+        shapes,
+        jsonKeys,
+        oddities: oddities(state, plan, run),
+    };
+}
+function jsonKeyProfiles(state) {
+    const byPath = new Map();
+    for (const entry of state.jsonRows ?? []) {
+        const known = byPath.get(entry.path) ?? { present: entry.present, types: new Set() };
+        known.types.add(entry.type);
+        byPath.set(entry.path, known);
+    }
+    return [...byPath.entries()].map(([path, entry]) => ({
+        path,
+        types: [...entry.types].sort(),
+        presentFraction: share(entry.present, state.nonNull),
+    }));
+}
+function oddities(state, plan, run) {
+    const found = [];
+    const where = plan.samplePercent === null ? '' : ` in a ${pct(plan.samplePercent / 100)} sample`;
+    const nulls = state.total - state.nonNull;
+    if (state.badCharacters > 0) {
+        found.push({
+            kind: 'invalid-encoding',
+            rowCount: state.badCharacters,
+            jsonPath: null,
+            types: null,
+            description: `${rowsDo(state.badCharacters, where, 'contains', 'contain')} control characters or the Unicode replacement character, a sign of text decoded with the wrong encoding.`,
+        });
+    }
+    if (!state.plan.catalog.notNull && nulls > 0 && state.total > 0 && nulls / state.total <= 0.01) {
+        found.push({
+            kind: 'unexpected-null',
+            rowCount: nulls,
+            jsonPath: null,
+            types: null,
+            description: `${rowsDo(nulls, where, 'is', 'are')} null while ${pct(state.nonNull / state.total)} of rows have a value.`,
+        });
+    }
+    if (state.nonNull > 0 && state.plan.family === 'text') {
+        const numericShare = state.numericText / state.nonNull;
+        const dateShare = state.dateText / state.nonNull;
+        if (numericShare >= 0.9 && state.numericText < state.nonNull) {
+            const odd = state.nonNull - state.numericText;
+            found.push({ kind: 'type-mismatch', rowCount: odd, jsonPath: null, types: { expected: 'number', found: ['text'] },
+                description: `${rowsDo(odd, where, 'is', 'are')} not a number while ${pct(numericShare)} of values are.` });
+        }
+        else if (dateShare >= 0.9 && state.dateText < state.nonNull) {
+            const odd = state.nonNull - state.dateText;
+            found.push({ kind: 'type-mismatch', rowCount: odd, jsonPath: null, types: { expected: 'iso-date', found: ['text'] },
+                description: `${rowsDo(odd, where, 'is', 'are')} not an ISO date while ${pct(dateShare)} of values are.` });
+        }
+    }
+    if (state.nonFinite > 0) {
+        found.push({ kind: 'type-mismatch', rowCount: state.nonFinite, jsonPath: null, types: { expected: 'finite', found: ['nan-or-infinity'] },
+            description: `${rowsDo(state.nonFinite, where, 'holds', 'hold')} NaN or infinity instead of a finite value.` });
+    }
+    if (state.jsonRoots !== null && state.nonNull > 0) {
+        const ranked = Object.entries(state.jsonRoots).sort((left, right) => right[1] - left[1]);
+        const [dominant, dominantRows] = ranked[0];
+        const others = ranked.slice(1).filter(([, rows]) => rows > 0);
+        const otherRows = others.reduce((sum, [, rows]) => sum + rows, 0);
+        if (otherRows > 0 && dominantRows / state.nonNull >= 0.9) {
+            found.push({
+                kind: 'type-mismatch',
+                rowCount: otherRows,
+                jsonPath: null,
+                types: { expected: dominant, found: others.map(([type]) => type) },
+                description: `${rowsDo(otherRows, where, 'holds', 'hold')} a JSON ${others.map(([type]) => type).join(' or ')} where ${pct(dominantRows / state.nonNull)} of rows hold a JSON ${dominant}.`,
+            });
+        }
+    }
+    if (state.oversize > 0) {
+        found.push({
+            kind: 'oversize',
+            rowCount: state.oversize,
+            jsonPath: null,
+            types: null,
+            description: `${rowsDo(state.oversize, where, 'is', 'are')} over 1 KiB and more than ten times the column's 95th-percentile length.`,
+        });
+    }
+    if (state.shapeRows !== null && state.nonNull > 0) {
+        const covered = state.shapeRows.reduce((sum, entry) => sum + entry.rows, 0);
+        if (covered < state.nonNull && covered / state.nonNull >= 0.9) {
+            const odd = state.nonNull - covered;
+            found.push({
+                kind: 'shape-outlier',
+                rowCount: odd,
+                jsonPath: null,
+                types: null,
+                description: `${rowsDo(odd, where, 'has', 'have')} a shape shared by fewer than ${run.categoryMinOwners} owners, while ${pct(covered / state.nonNull)} of values have one of the column's common shapes.`,
+            });
+        }
+    }
+    if (state.jsonRows !== null && state.jsonRoots !== null) {
+        const objects = state.jsonRoots.object;
+        const byPath = new Map();
+        for (const entry of state.jsonRows) {
+            const known = byPath.get(entry.path) ?? { present: entry.present, types: new Map() };
+            known.types.set(entry.type, entry.rows);
+            byPath.set(entry.path, known);
+        }
+        for (const [path, entry] of byPath) {
+            const topLevel = !path.includes('.') && !path.includes('[]');
+            if (topLevel && objects > 0 && entry.present < objects && entry.present / objects >= 0.9) {
+                const missing = objects - entry.present;
+                found.push({
+                    kind: 'missing-json-key',
+                    rowCount: missing,
+                    jsonPath: path,
+                    types: null,
+                    description: `${rowsDo(missing, where, 'is', 'are')} missing the key "${path}" that ${pct(entry.present / objects)} of JSON objects have.`,
+                });
+            }
+            const typed = [...entry.types.entries()].filter(([type]) => type !== 'null').sort((left, right) => right[1] - left[1]);
+            if (typed.length > 1 && typed[0][1] / entry.present >= 0.9) {
+                const odd = typed.slice(1).reduce((sum, [, rows]) => sum + rows, 0);
+                found.push({
+                    kind: 'type-mismatch',
+                    rowCount: odd,
+                    jsonPath: path,
+                    types: { expected: typed[0][0], found: typed.slice(1).map(([type]) => type) },
+                    description: `${rowsDo(odd, where, 'holds', 'hold')} "${path}" as a JSON ${typed.slice(1).map(([type]) => type).join(' or ')} where ${pct(typed[0][1] / entry.present)} hold it as a ${typed[0][0]}.`,
+                });
+            }
+            const nullRows = entry.types.get('null') ?? 0;
+            if (nullRows > 0 && nullRows / entry.present <= 0.01) {
+                found.push({
+                    kind: 'unexpected-null',
+                    rowCount: nullRows,
+                    jsonPath: path,
+                    types: null,
+                    description: `${rowsDo(nullRows, where, 'has', 'have')} "${path}" set to JSON null while ${pct(1 - nullRows / entry.present)} of rows with the key have a value.`,
+                });
+            }
+        }
+    }
+    return found;
+}
+/* ------------------------------------------------------------ command */
+function writeProfileFile(path, profile) {
+    const target = resolve(path);
+    const temporary = `${target}.tmp-${process.pid}`;
+    writeFileSync(temporary, `${JSON.stringify(profile, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 });
+    renameSync(temporary, target);
+}
+export async function databaseProfileCommand(options) {
+    const settings = parseDatabaseProfileSettings(options);
+    const connectionString = readConnectionString(settings.urlEnv);
+    const db = new ReadOnlyDatabase(connectionString, settings.concurrency, settings.statementTimeoutMs);
+    const log = (line) => { process.stderr.write(`${line}\n`); };
+    let profile;
+    try {
+        profile = await profileDatabase(db, settings, log);
+    }
+    finally {
+        await db.close();
+    }
+    writeProfileFile(settings.out, profile);
+    const columns = profile.tables.flatMap(table => table.columns);
+    const kept = columns.filter(column => (column.categories ?? []).length > 0).length;
+    const personal = columns.filter(column => column.kind === 'personal' || column.kind === 'free-text').length;
+    console.log(chalk.green(`Wrote ${settings.out}: ${profile.tables.length} tables, ${columns.length} columns; ${kept} keep category values, ${personal} are personal (values never leave).`));
+    console.log(`Review exactly what would be sent:  haystack db profile show ${settings.out}`);
+}