@haystackeditor/cli 0.17.0 → 0.18.0

package/dist/commands/db-profile-sql-literals.js

@@ -0,0 +1,140 @@
+/**
+ * Replaces every literal in a piece of Postgres SQL text with `?`, for index
+ * definitions: a partial-index predicate or an index expression can hold a
+ * value (`WHERE email <> 'ceo@example.com'`, `WHERE tenant_id = 4711`).
+ *
+ * Approach: a lexer, not a pattern. The text is read left to right with the
+ * same token rules as the Postgres lexer, so every character is known to be
+ * inside a quoted identifier, a string, a number or plain syntax before
+ * anything is replaced:
+ *   - quoted identifiers ("Mixed Case", with "" escapes) are copied untouched,
+ *     so a name that looks like a string or holds digits is never mangled;
+ *   - unquoted identifiers and keywords (col1, idx_2024, text_pattern_ops) are
+ *     read whole, so the digits inside them are never taken for numbers;
+ *   - strings become `?`: '...' with '' escapes, E'...' with backslash escapes,
+ *     B'...' / X'...' bit strings, N'...', U&'...' and $tag$...$tag$;
+ *   - numbers (42, 27182.8182, 1e300, 0x1F, 1_000) become `?`.
+ * Everything else (operators, parentheses, `::` and the type names after it)
+ * is copied. So `'2024-01-01'::date` becomes `?::date`: the value goes, the
+ * type (schema, not data) stays; `ARRAY['a'::text, 'b'::text]` becomes
+ * `ARRAY[?::text, ?::text]`; `'{1,2}'::integer[]` becomes `?::integer[]`.
+ * Numbers in type modifiers (`numeric(10,2)`) and storage options
+ * (`fillfactor='70'`) are replaced too: over-redacting schema is harmless.
+ * The keywords TRUE, FALSE and NULL are kept; they cannot carry a value.
+ *
+ * Postgres never prints a comment in a definition, and a comment could hide
+ * text this lexer does not read, so a comment (or an unterminated string or
+ * quoted identifier) is refused instead of guessed at.
+ */
+export class SqlLiteralError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'SqlLiteralError';
+    }
+}
+const IDENT_START = /[A-Za-z_\u0080-￿]/u;
+const IDENT_PART = /[A-Za-z0-9_$\u0080-￿]/u;
+const DIGIT = /[0-9]/u;
+const NUMBER = /^(?:0[xX][0-9A-Fa-f_]+|0[oO][0-7_]+|0[bB][01_]+|(?:[0-9][0-9_]*(?:\.[0-9_]*)?|\.[0-9][0-9_]*)(?:[eE][+-]?[0-9]+)?)/u;
+const DOLLAR_TAG = /^\$(?:[A-Za-z_\u0080-￿][A-Za-z0-9_\u0080-￿]*)?\$/u;
+/** End (exclusive) of a quote-delimited token starting at `start` (the
+ * opening quote). A doubled quote is an escaped quote; with `backslashes`, a
+ * backslash escapes the next character (E'...' strings). */
+function quotedEnd(text, start, quote, backslashes, what) {
+    let index = start + 1;
+    while (index < text.length) {
+        const char = text[index];
+        if (backslashes && char === '\\') {
+            index += 2;
+            continue;
+        }
+        if (char === quote) {
+            if (text[index + 1] === quote) {
+                index += 2;
+                continue;
+            }
+            return index + 1;
+        }
+        index += 1;
+    }
+    throw new SqlLiteralError(`it has an unterminated ${what}`);
+}
+/** `text` with every literal replaced by `?`. Throws SqlLiteralError when the
+ * text holds something the lexer refuses to guess about. */
+export function redactSqlLiterals(text) {
+    let out = '';
+    let index = 0;
+    while (index < text.length) {
+        const char = text[index];
+        const next = text[index + 1] ?? '';
+        if ((char === '-' && next === '-') || (char === '/' && next === '*')) {
+            throw new SqlLiteralError('it contains a comment, which Postgres never prints in a definition');
+        }
+        if (char === '"') {
+            const end = quotedEnd(text, index, '"', false, 'quoted identifier');
+            out += text.slice(index, end);
+            index = end;
+            continue;
+        }
+        if (char === "'") {
+            index = quotedEnd(text, index, "'", false, 'string');
+            out += '?';
+            continue;
+        }
+        if (char === '$') {
+            const tag = text.slice(index).match(DOLLAR_TAG)?.[0];
+            if (tag !== undefined) {
+                const close = text.indexOf(tag, index + tag.length);
+                if (close < 0)
+                    throw new SqlLiteralError('it has an unterminated dollar-quoted string');
+                index = close + tag.length;
+                out += '?';
+                continue;
+            }
+            // A parameter ($1) never appears in a definition; copy it whole.
+            const parameter = text.slice(index).match(/^\$[0-9]+/u)?.[0] ?? '$';
+            out += parameter;
+            index += parameter.length;
+            continue;
+        }
+        if (IDENT_START.test(char)) {
+            let end = index + 1;
+            while (end < text.length && IDENT_PART.test(text[end]))
+                end += 1;
+            const word = text.slice(index, end);
+            const upper = word.toUpperCase();
+            if (text[end] === "'" && (upper === 'E' || upper === 'B' || upper === 'X' || upper === 'N')) {
+                index = quotedEnd(text, end, "'", upper === 'E', 'string');
+                out += '?';
+                continue;
+            }
+            if (upper === 'U' && text[end] === '&' && (text[end + 1] === "'" || text[end + 1] === '"')) {
+                const quote = text[end + 1];
+                const close = quotedEnd(text, end + 1, quote, false, quote === '"' ? 'quoted identifier' : 'string');
+                out += quote === '"' ? text.slice(index, close) : '?';
+                index = close;
+                continue;
+            }
+            out += word;
+            index = end;
+            continue;
+        }
+        const previous = index === 0 ? '' : text[index - 1];
+        const startsNumber = DIGIT.test(char)
+            || (char === '.' && DIGIT.test(next) && !(IDENT_PART.test(previous) || previous === '"' || previous === ')' || previous === ']'));
+        if (startsNumber) {
+            const number = text.slice(index).match(NUMBER)?.[0];
+            if (number === undefined)
+                throw new SqlLiteralError(`it has a malformed number at character ${index + 1}`);
+            const after = text[index + number.length] ?? '';
+            if (IDENT_PART.test(after))
+                throw new SqlLiteralError(`it has a number followed by letters at character ${index + 1}`);
+            out += '?';
+            index += number.length;
+            continue;
+        }
+        out += char;
+        index += 1;
+    }
+    return out;
+}

package/dist/commands/db-profile-sql.js

@@ -0,0 +1,362 @@
+/**
+ * SQL for `haystack db profile`. Every statement here is a single SELECT (or
+ * WITH ... SELECT) that returns aggregates: counts, fractions, lengths,
+ * quantiles. The only strings a statement may return are ones that already
+ * passed an owner threshold in the same statement (a `HAVING` on the number of
+ * distinct owners), or values the schema itself declares (enum labels and
+ * CHECK value lists, filtered with `= ANY($1)`).
+ *
+ * Identifiers come from the catalog and are always quoted; numbers embedded in
+ * SQL are produced here from validated inputs. Thresholds travel as parameters.
+ */
+import { averageWordsSql, fourWordsPredicate, KEPT_VALUE_PERSONAL_SHAPES, shapeSql, VALUE_SHAPE_TESTS, } from './db-profile-privacy.js';
+export function quoteIdent(name) {
+    if (name.length === 0 || name.includes('\u0000'))
+        throw new Error(`Refusing to quote an empty or NUL-containing identifier.`);
+    return `"${name.replace(/"/gu, '""')}"`;
+}
+export function qualified(ref) {
+    return `${quoteIdent(ref.schema)}.${quoteIdent(ref.table)}`;
+}
+function sqlNumber(value, label) {
+    if (!Number.isFinite(value))
+        throw new Error(`${label} must be finite to embed in SQL.`);
+    return String(value);
+}
+/** Seed for TABLESAMPLE ... REPEATABLE, so every query in a run (and a re-run
+ * on unchanged data) reads the same sampled pages. */
+export const SAMPLE_SEED = 20260924;
+export const QUANTILE_POINTS = [0.05, 0.25, 0.5, 0.75, 0.95];
+const QUANTILE_ARRAY = `ARRAY[${QUANTILE_POINTS.join(', ')}]::float8[]`;
+function tableSample(samplePercent) {
+    if (samplePercent === null)
+        return '';
+    if (!(samplePercent > 0 && samplePercent < 100))
+        throw new Error(`Sample percentage ${samplePercent} is outside (0, 100).`);
+    return ` TABLESAMPLE SYSTEM (${sqlNumber(Number(samplePercent.toFixed(6)), 'sample percentage')}) REPEATABLE (${SAMPLE_SEED})`;
+}
+function columnTuple(alias, columns) {
+    if (columns.length === 1)
+        return `${alias}.${quoteIdent(columns[0])}`;
+    // A multi-column key with any null part references nothing, so it owns nothing.
+    const nullCheck = columns.map(column => `${alias}.${quoteIdent(column)} IS NULL`).join(' OR ');
+    return `CASE WHEN ${nullCheck} THEN NULL ELSE ROW(${columns.map(column => `${alias}.${quoteIdent(column)}`).join(', ')}) END`;
+}
+function joinOn(childAlias, parentAlias, edge) {
+    return edge.columns
+        .map((column, index) => `${parentAlias}.${quoteIdent(edge.referencedColumns[index])} = ${childAlias}.${quoteIdent(column)}`)
+        .join(' AND ');
+}
+/** The table alone as `t0` (sampled or not). ONLY: a table with
+ * legacy-inheritance children is measured on its own rows. */
+function tableFrom(table, samplePercent) {
+    return `ONLY ${qualified(table)} AS t0${tableSample(samplePercent)}`;
+}
+/** `FROM` clause of a table (sampled or not) joined along each owner path. */
+export function scanSql(source) {
+    let from = tableFrom(source.table, source.samplePercent);
+    const owners = [];
+    if (source.reach.kind === 'paths') {
+        source.reach.paths.forEach((path, pathIndex) => {
+            let childAlias = 't0';
+            // Join every table on the path, the owner table included, and count the
+            // owner table's own key: a key no owner row holds (an orphan under a
+            // NOT VALID foreign key) is null, so it is no owner. Each join is on a
+            // primary or unique key, so it never repeats a row.
+            path.forEach((edge, hopIndex) => {
+                const alias = `p${pathIndex}h${hopIndex}`;
+                from += ` LEFT JOIN ${qualified(edge.to)} AS ${alias} ON ${joinOn(childAlias, alias, edge)}`;
+                childAlias = alias;
+            });
+            owners.push(columnTuple(childAlias, path[path.length - 1].referencedColumns));
+        });
+    }
+    return { from, owners };
+}
+/** Distinct owners among the rows selected by `filter` (all rows when null).
+ * With no owner expressions (rows are owners, or this is the owner table) it
+ * counts rows. A table the owner table is unreachable from also has none, and
+ * the profiler withholds every owner-gated number for it. */
+function ownerCount(scan, filter, ownerRefs = scan.owners) {
+    const filterSql = filter === null ? '' : ` FILTER (WHERE ${filter})`;
+    if (ownerRefs.length === 0)
+        return `count(*)${filterSql}`;
+    const counts = ownerRefs.map(owner => `count(DISTINCT ${owner})${filterSql}`);
+    return counts.length === 1 ? counts[0] : `LEAST(${counts.join(', ')})`;
+}
+export function columnRef(name) {
+    return `t0.${quoteIdent(name)}`;
+}
+export function hasLengthQuantiles(family) {
+    return family === 'text' || family === 'json' || family === 'binary';
+}
+export function hasNumberQuantiles(family) {
+    return family === 'number' || family === 'time';
+}
+function byteLengthSql(column) {
+    const ref = columnRef(column.name);
+    return column.family === 'binary' ? `octet_length(${ref})` : `octet_length(${ref}::text)`;
+}
+function numberSql(column) {
+    const ref = columnRef(column.name);
+    if (column.family === 'time')
+        return `extract(epoch FROM ${ref})::float8`;
+    if (column.typname === 'money')
+        return `${ref}::numeric::float8`;
+    return `${ref}::float8`;
+}
+const NON_FINITE = `('Infinity'::float8, '-Infinity'::float8, 'NaN'::float8)`;
+const NUMERIC_TEXT = `'^[[:space:]]*[-+]?[0-9]+(\\.[0-9]+)?[[:space:]]*$'`;
+const ISO_DATE_TEXT = `'^[0-9]{4}-[0-9]{2}-[0-9]{2}'`;
+const BAD_CHARACTERS = `'[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f]'`;
+export const JSON_ROOT_TYPES = ['object', 'array', 'string', 'number', 'boolean', 'null'];
+/** What `count(DISTINCT ...)` counts for a column read in full. A type with
+ * no default btree operator class (varchar has none of its own) is counted by
+ * its text, which is exact for text and never fewer than the true count for
+ * numbers, since the count decides category columns. Other such types (JSON,
+ * geometry) are counted by a 64-bit hash of their text. */
+function exactDistinctValue(column) {
+    const ref = columnRef(column.name);
+    if (column.sortable)
+        return ref;
+    if (column.family === 'text' || column.family === 'number')
+        return `${ref}::text`;
+    return `hashtextextended(${ref}::text, 0)`;
+}
+/**
+ * One pass over the table (or its sample) computing, for every column, the
+ * numbers the classification and the profile need. Result keys are
+ * `c<index>_<measure>`.
+ */
+export function statsQuery(scan, columns, exactDistinct) {
+    const parts = ['count(*)::float8 AS total'];
+    columns.forEach(({ index, column }) => {
+        const ref = columnRef(column.name);
+        const key = `c${index}`;
+        parts.push(`count(${ref})::float8 AS ${key}_nn`);
+        if (exactDistinct)
+            parts.push(`count(DISTINCT ${exactDistinctValue(column)})::float8 AS ${key}_nd`);
+        parts.push(`avg(pg_column_size(${ref}))::float8 AS ${key}_aw`);
+        parts.push(`(${ownerCount(scan, `${ref} IS NOT NULL`)})::float8 AS ${key}_own`);
+        if (hasLengthQuantiles(column.family)) {
+            parts.push(`percentile_cont(${QUANTILE_ARRAY}) WITHIN GROUP (ORDER BY ${byteLengthSql(column)}) AS ${key}_lq`);
+        }
+        if (hasNumberQuantiles(column.family)) {
+            const value = numberSql(column);
+            parts.push(`percentile_cont(${QUANTILE_ARRAY}) WITHIN GROUP (ORDER BY ${value}) FILTER (WHERE ${value} NOT IN ${NON_FINITE}) AS ${key}_nq`);
+            parts.push(`count(*) FILTER (WHERE ${value} IN ${NON_FINITE})::float8 AS ${key}_nonfinite`);
+        }
+        if (column.family === 'text') {
+            const value = `${ref}::text`;
+            for (const test of VALUE_SHAPE_TESTS) {
+                parts.push(`count(*) FILTER (WHERE ${test.predicate(value)})::float8 AS ${key}_t_${test.id.replace('-', '_')}`);
+            }
+            parts.push(`${averageWordsSql(value)}::float8 AS ${key}_words`);
+            parts.push(`count(*) FILTER (WHERE ${fourWordsPredicate(value)})::float8 AS ${key}_w4`);
+            parts.push(`count(*) FILTER (WHERE ${value} ~ ${NUMERIC_TEXT})::float8 AS ${key}_numeric`);
+            parts.push(`count(*) FILTER (WHERE ${value} ~ ${ISO_DATE_TEXT})::float8 AS ${key}_date`);
+            parts.push(`count(*) FILTER (WHERE strpos(${value}, chr(65533)) > 0 OR ${value} ~ ${BAD_CHARACTERS})::float8 AS ${key}_bad`);
+        }
+        if (column.family === 'json') {
+            for (const type of JSON_ROOT_TYPES) {
+                parts.push(`count(*) FILTER (WHERE jsonb_typeof(${ref}::jsonb) = '${type}')::float8 AS ${key}_j_${type}`);
+            }
+        }
+        if (column.family === 'boolean') {
+            parts.push(`count(*) FILTER (WHERE ${ref}::boolean)::float8 AS ${key}_true`);
+        }
+    });
+    return `SELECT ${parts.join(',\n  ')}\nFROM ${scan.from}`;
+}
+/** Select-list items `statsQuery` emits for a column. */
+export function statsItemCount(column, exactDistinct) {
+    return 3 + (exactDistinct ? 1 : 0)
+        + (hasLengthQuantiles(column.family) ? 1 : 0)
+        + (hasNumberQuantiles(column.family) ? 2 : 0)
+        + (column.family === 'text' ? VALUE_SHAPE_TESTS.length + 5 : 0)
+        + (column.family === 'json' ? JSON_ROOT_TYPES.length : 0)
+        + (column.family === 'boolean' ? 1 : 0);
+}
+/** Postgres allows 1,664 select-list entries; wide tables are measured in
+ * several passes of at most this many (the same sample pages each time). */
+export const STATS_ITEMS_PER_QUERY = 1_000;
+export function statsBatches(columns, exactDistinct) {
+    const batches = [[]];
+    let items = 1;
+    columns.forEach((column, index) => {
+        const needed = statsItemCount(column, exactDistinct);
+        if (items + needed > STATS_ITEMS_PER_QUERY && batches[batches.length - 1].length > 0) {
+            batches.push([]);
+            items = 1;
+        }
+        batches[batches.length - 1].push({ index, column });
+        items += needed;
+    });
+    return batches;
+}
+/**
+ * For one column of a sampled table: how many distinct values the sample
+ * holds on exactly `pages` sampled pages, one row per page count that occurs
+ * (see `db-profile-distinct.ts`). Only counts leave the database. A value is
+ * compared as itself, or by a 64-bit hash of its text when its type has no
+ * default btree operator class; a hash collision can only lower the counts.
+ */
+export function sampleDistinctQuery(table, samplePercent, column) {
+    const ref = columnRef(column.name);
+    const value = column.sortable ? ref : `hashtextextended(${ref}::text, 0)`;
+    return `SELECT f.pages::float8 AS pages, count(*)::float8 AS value_count
+FROM (SELECT p.v, count(*) AS pages
+      FROM (SELECT DISTINCT ${value} AS v, (t0.ctid::text::point)[0] AS page
+            FROM ${tableFrom(table, samplePercent)}
+            WHERE ${ref} IS NOT NULL) p
+      GROUP BY p.v) f
+GROUP BY f.pages
+ORDER BY f.pages`;
+}
+/**
+ * Exact distinct non-null counts of several columns over the WHOLE table, in
+ * one read: each grouping set holds one column's distinct values, and in the
+ * other sets' rows that column is null, so `count(x.cN)` counts exactly its
+ * own distinct non-null values. Result keys are `c<index>_d`.
+ */
+export function categoryDistinctQuery(table, columns) {
+    if (columns.length === 0)
+        throw new Error('categoryDistinctQuery needs at least one column.');
+    const counts = columns.map(({ index }) => `count(x.c${index})::float8 AS c${index}_d`).join(',\n  ');
+    const select = columns.map(({ index, column }) => `${columnRef(column.name)} AS c${index}`).join(', ');
+    const sets = columns.map(({ column }) => `(${columnRef(column.name)})`).join(', ');
+    return `SELECT ${counts}
+FROM (SELECT ${select}
+      FROM ${tableFrom(table, null)}
+      GROUP BY GROUPING SETS (${sets})) x`;
+}
+/** Rows whose byte length exceeds a per-column threshold; one pass per table. */
+export function oversizeQuery(scan, checks) {
+    const parts = checks.map(({ index, column, thresholdBytes }) => `count(*) FILTER (WHERE ${byteLengthSql(column)} > ${sqlNumber(Math.round(thresholdBytes), 'oversize threshold')})::float8 AS c${index}_big`);
+    return `SELECT ${parts.join(',\n  ')}\nFROM ${scan.from}`;
+}
+/* ------------------------------------------------------------ values */
+/** Data values of a category column, each returned only when it is shared by
+ * at least $1 owners. */
+export function categoryValuesQuery(scan, column) {
+    const ref = columnRef(column);
+    return `SELECT ${ref}::text AS value, count(*)::float8 AS rows
+FROM ${scan.from}
+WHERE ${ref} IS NOT NULL
+GROUP BY ${ref}
+HAVING ${ownerCount(scan, null)} >= $1
+ORDER BY rows DESC, value`;
+}
+/** Row counts for values the schema declares ($1, a text array). */
+export function schemaValuesQuery(scan, column) {
+    const ref = columnRef(column);
+    return `SELECT ${ref}::text AS value, count(*)::float8 AS rows
+FROM ${scan.from}
+WHERE ${ref}::text = ANY($1::text[])
+GROUP BY 1`;
+}
+function ownerColumns(scan) {
+    const select = scan.owners.map((owner, index) => `, ${owner} AS o${index}`).join('');
+    return { select, refs: scan.owners.map((_, index) => `x.o${index}`) };
+}
+/** Character-class shapes shared by at least $1 owners. */
+export function shapesQuery(scan, column) {
+    const ref = columnRef(column);
+    const owners = ownerColumns(scan);
+    return `SELECT x.shape, count(*)::float8 AS rows
+FROM (SELECT ${shapeSql(`${ref}::text`)} AS shape${owners.select}
+      FROM ${scan.from}
+      WHERE ${ref} IS NOT NULL) x
+GROUP BY x.shape
+HAVING ${ownerCount(scan, null, owners.refs)} >= $1
+ORDER BY rows DESC, x.shape`;
+}
+/** Deepest JSON key path followed; deeper structure is summarised by its parent. */
+export const JSON_KEY_MAX_DEPTH = 6;
+/**
+ * JSON key paths (objects as `a.b`, array elements as `a[]`) shared by at
+ * least $1 owners, with the JSON types seen at each path and how many rows
+ * carry each type. A path is never returned, whatever its owner count, when
+ * any key on it (tested one key at a time, since a key may itself hold a
+ * dot) or the whole path has a personal value shape (KEPT_VALUE_PERSONAL_SHAPES:
+ * an email, phone number, IP address, government id, card number or secret):
+ * `15551234567.verified` is withheld with `15551234567`.
+ */
+export function jsonKeysQuery(scan, column) {
+    const ref = columnRef(column);
+    const owners = ownerColumns(scan);
+    const ownerNames = scan.owners.map((_, index) => `o${index}`);
+    const carry = ownerNames.map(name => `, w.${name}`).join('');
+    const baseOwners = ownerNames.map(name => `, b.${name}`).join('');
+    const walkOwners = ownerNames.map(name => `, ${name}`).join('');
+    const pathOwnerCount = ownerNames.length === 0
+        ? 'count(DISTINCT w.rid)'
+        : ownerNames.length === 1
+            ? `count(DISTINCT w.${ownerNames[0]})`
+            : `LEAST(${ownerNames.map(name => `count(DISTINCT w.${name})`).join(', ')})`;
+    const personal = (value) => VALUE_SHAPE_TESTS
+        .filter(test => KEPT_VALUE_PERSONAL_SHAPES.some(shape => shape.id === test.id))
+        .map(test => test.predicate(value))
+        .join(' OR ');
+    return `WITH RECURSIVE base AS (
+  SELECT t0.ctid AS rid${owners.select}, ${ref}::jsonb AS doc
+  FROM ${scan.from}
+  WHERE ${ref} IS NOT NULL
+), walk(rid${walkOwners}, path, value, depth, personal) AS (
+  SELECT b.rid${baseOwners}, ''::text, b.doc, 0, false FROM base b
+  UNION ALL
+  SELECT w.rid${carry}, child.path, child.value, w.depth + 1, w.personal OR child.personal
+  FROM walk w CROSS JOIN LATERAL (
+    SELECT CASE WHEN w.path = '' THEN e.key ELSE w.path || '.' || e.key END AS path, e.value, (${personal('e.key')}) AS personal
+    FROM jsonb_each(CASE WHEN jsonb_typeof(w.value) = 'object' THEN w.value ELSE '{}'::jsonb END) e
+    UNION ALL
+    SELECT w.path || '[]', a.value, false
+    FROM jsonb_array_elements(CASE WHEN jsonb_typeof(w.value) = 'array' THEN w.value ELSE '[]'::jsonb END) a
+  ) child
+  WHERE w.depth < ${JSON_KEY_MAX_DEPTH}
+), clean AS (
+  SELECT w.* FROM walk w
+  WHERE w.depth > 0 AND NOT w.personal AND NOT (${personal('w.path')})
+), kept AS (
+  SELECT w.path, count(DISTINCT w.rid)::float8 AS present
+  FROM clean w
+  GROUP BY w.path
+  HAVING ${pathOwnerCount} >= $1
+)
+SELECT k.path, k.present, jsonb_typeof(w.value) AS type, count(DISTINCT w.rid)::float8 AS rows
+FROM clean w JOIN kept k ON k.path = w.path
+GROUP BY k.path, k.present, jsonb_typeof(w.value)
+ORDER BY k.path, type`;
+}
+/* ------------------------------------------------------------ foreign keys */
+/** Children per parent over the (sampled) parent table, counting parents
+ * with no children, as quantiles; and how many of those parents have at least
+ * one child, which for a validated single-column key is the number of
+ * distinct values the child's column holds (exactly, when the parent is read
+ * in full). */
+export function childrenPerParentQuery(edge, parentSamplePercent) {
+    const parentKeyNotNull = edge.referencedColumns.map(column => `p.${quoteIdent(column)} IS NOT NULL`).join(' AND ');
+    const groupBy = edge.referencedColumns.map(column => `p.${quoteIdent(column)}`).join(', ');
+    const on = edge.columns
+        .map((column, index) => `c.${quoteIdent(column)} = p.${quoteIdent(edge.referencedColumns[index])}`)
+        .join(' AND ');
+    return `SELECT percentile_cont(${QUANTILE_ARRAY}) WITHIN GROUP (ORDER BY x.children) AS q, count(*)::float8 AS parents,
+  count(*) FILTER (WHERE x.children > 0)::float8 AS referenced
+FROM (SELECT count(c.ctid)::float8 AS children
+      FROM ${qualified(edge.to)} AS p${tableSample(parentSamplePercent)}
+      LEFT JOIN ONLY ${qualified(edge.from)} AS c ON ${on}
+      WHERE ${parentKeyNotNull}
+      GROUP BY ${groupBy}) x`;
+}
+/** Share of (sampled) child rows whose non-null key has no parent. Only run
+ * for NOT VALID constraints; a validated constraint guarantees zero. */
+export function orphanQuery(edge, childSamplePercent) {
+    const childKeyNotNull = edge.columns.map(column => `c.${quoteIdent(column)} IS NOT NULL`).join(' AND ');
+    const match = edge.columns
+        .map((column, index) => `p.${quoteIdent(edge.referencedColumns[index])} = c.${quoteIdent(column)}`)
+        .join(' AND ');
+    return `SELECT count(*)::float8 AS children,
+  count(*) FILTER (WHERE NOT EXISTS (SELECT 1 FROM ${qualified(edge.to)} AS p WHERE ${match}))::float8 AS orphans
+FROM ONLY ${qualified(edge.from)} AS c${tableSample(childSamplePercent)}
+WHERE ${childKeyNotNull}`;
+}

package/dist/commands/db-profile-upload.js

@@ -0,0 +1,88 @@
+/**
+ * `haystack db profile upload <file> --repo owner/repo [--dependency <id>]` —
+ * send a reviewed profile to `POST /api/agent/cloud-verifier/database-profiles`
+ * as a `DatabaseProfileUploadRequestV1`, authenticated with the caller's
+ * Haystack login like every other CLI call, and print the stored
+ * `DatabaseProfileUploadV1`. Without --dependency the profile is the
+ * repository's: onboarding builds the run map's Postgres stand-in from it when
+ * the run map declares exactly one, whatever id it gave that dependency.
+ *
+ * The file is parsed with the same strict parser `show` uses, so what is sent
+ * is exactly what `show` printed.
+ */
+import chalk from 'chalk';
+import { DATABASE_PROFILE_UPLOAD_REQUEST_VERSION, parseDatabaseProfileUpload, } from './db-profile-contract.js';
+import { readDatabaseProfileFile } from './db-profile-show.js';
+import { resolveAuthContext } from '../utils/auth.js';
+import { classifyHttpError, haystackApiUrl } from '../utils/haystack-api.js';
+export const DATABASE_PROFILES_PATH = '/api/agent/cloud-verifier/database-profiles';
+/** GitHub's owner and repository name alphabets. */
+const REPOSITORY = /^([A-Za-z0-9](?:[A-Za-z0-9-]{0,38}))\/([A-Za-z0-9_.-]{1,100})$/u;
+/** Run-map dependency ids are stable kebab-case ids (the same check as
+ * `requireId` in verifier/front-half/src/derivations/surface-map.ts). */
+const DEPENDENCY_ID = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u;
+export function buildDatabaseProfileUploadRequest(file, options) {
+    if (options.repo === undefined)
+        throw new Error('--repo is required: the GitHub repository this database belongs to, as owner/repo.');
+    const repository = options.repo.match(REPOSITORY);
+    if (!repository || repository[2] === '.' || repository[2] === '..') {
+        throw new Error(`--repo must be a GitHub owner/repo (got "${options.repo}").`);
+    }
+    if (options.dependency !== undefined && !DEPENDENCY_ID.test(options.dependency)) {
+        throw new Error(`--dependency must be a run-map dependency id in kebab-case, e.g. app-postgres (got "${options.dependency}").`);
+    }
+    const profile = readDatabaseProfileFile(file);
+    return {
+        owner: repository[1],
+        name: repository[2],
+        request: {
+            version: DATABASE_PROFILE_UPLOAD_REQUEST_VERSION,
+            repository: options.repo,
+            ...(options.dependency === undefined ? {} : { dependencyId: options.dependency }),
+            profile,
+        },
+    };
+}
+export async function uploadDatabaseProfile(request, token) {
+    const response = await fetch(haystackApiUrl(DATABASE_PROFILES_PATH), {
+        method: 'POST',
+        headers: {
+            Authorization: `Bearer ${token}`,
+            Accept: 'application/json',
+            'User-Agent': 'Haystack-CLI',
+            'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(request),
+        signal: AbortSignal.timeout(120_000),
+    });
+    if (!response.ok)
+        throw await classifyHttpError(response, `Haystack API ${DATABASE_PROFILES_PATH}`);
+    let body;
+    try {
+        body = await response.json();
+    }
+    catch (error) {
+        throw new Error(`Haystack API ${DATABASE_PROFILES_PATH} returned a body that is not JSON: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    const upload = parseDatabaseProfileUpload(body);
+    const expected = request.dependencyId ?? null;
+    if (upload.dependencyId !== expected) {
+        const describe = (id) => (id === null ? 'the repository' : `dependency ${id}`);
+        throw new Error(`Haystack API ${DATABASE_PROFILES_PATH} stored the profile for ${describe(upload.dependencyId)}, not ${describe(expected)}.`);
+    }
+    return upload;
+}
+export async function databaseProfileUploadCommand(file, options) {
+    const { owner, name, request } = buildDatabaseProfileUploadRequest(file, options);
+    const auth = await resolveAuthContext({ preferredLogin: options.account, owner, repo: name });
+    const upload = await uploadDatabaseProfile(request, auth.token);
+    if (upload.dependencyId === null) {
+        console.log(chalk.green(`Uploaded ${file} for ${request.repository}.`));
+        console.log('  Onboarding builds the stand-in database from it when the run map declares exactly one Postgres dependency;');
+        console.log('  otherwise haystack verify hosted start says which ids it declares, and you upload again with --dependency <id>.');
+    }
+    else {
+        console.log(chalk.green(`Uploaded ${file} for ${request.repository}, dependency ${upload.dependencyId}.`));
+    }
+    console.log(`  stored as ${upload.profile.artifactId} (sha256 ${upload.profile.sha256}) at ${upload.uploadedAt}`);
+}