@haystackeditor/cli 0.17.0 → 0.18.0

package/dist/commands/db-profile-privacy.js

@@ -0,0 +1,450 @@
+export function typeFamily(base) {
+    if (base.typtype === 'e')
+        return 'enum';
+    if (base.typname === 'json' || base.typname === 'jsonb')
+        return 'json';
+    if (base.typname === 'bytea')
+        return 'binary';
+    if (base.typname === 'uuid')
+        return 'uuid';
+    if (['inet', 'cidr', 'macaddr', 'macaddr8'].includes(base.typname))
+        return 'network';
+    switch (base.typcategory) {
+        case 'S': return 'text';
+        case 'N': return 'number';
+        case 'D':
+        case 'T': return 'time';
+        case 'B': return 'boolean';
+        case 'I': return 'network';
+        default: return 'other';
+    }
+}
+/* ------------------------------------------------------------ column names */
+/** Split `shippingAddress_line1` into `shipping`, `address`, `line`, `1`. */
+export function columnNameTokens(name) {
+    return name
+        .replace(/([a-z0-9])([A-Z])/gu, '$1 $2')
+        .replace(/([A-Za-z])([0-9])/gu, '$1 $2')
+        .toLowerCase()
+        .split(/[^a-z0-9]+/u)
+        .filter(token => token.length > 0);
+}
+/** Ordered: the first rule that matches gives the reason. */
+export const PERSONAL_NAME_RULES = [
+    {
+        kind: 'personal', what: 'a password, secret, token or payment credential',
+        substrings: ['password', 'passwd', 'passphrase', 'secret', 'apikey', 'privatekey', 'accesskey', 'credential',
+            'creditcard', 'cardnumber', 'accountnumber', 'routingnumber', 'sortcode', 'iban'],
+        tokens: ['token', 'tokens', 'pwd', 'pin', 'otp', 'salt', 'cvv', 'cvc', 'jwt', 'session', 'cookie'],
+    },
+    {
+        kind: 'personal', what: 'a government identifier',
+        substrings: ['socialsecurity', 'passport', 'nationalid', 'nationalinsurance', 'taxid', 'taxnumber',
+            'driverslicense', 'driverlicense', 'driverslicence', 'driverlicence', 'licensenumber', 'licencenumber',
+            'identitycard', 'identitynumber', 'idcard'],
+        tokens: ['ssn', 'sin', 'nin', 'tin', 'aadhaar', 'cpf', 'nhs'],
+    },
+    {
+        kind: 'personal', what: 'a date of birth',
+        substrings: ['birth'],
+        tokens: ['dob', 'birthday', 'bday', 'born'],
+    },
+    {
+        kind: 'personal', what: 'an email address',
+        substrings: ['email'],
+        tokens: ['mail'],
+    },
+    {
+        kind: 'personal', what: 'a phone number',
+        substrings: ['phone', 'mobile', 'msisdn', 'whatsapp'],
+        tokens: ['tel', 'fax', 'cell', 'sms'],
+    },
+    {
+        kind: 'personal', what: 'a network address or device identifier',
+        substrings: ['ipaddress', 'ipaddr', 'macaddress', 'useragent', 'remoteaddr', 'hostname'],
+        tokens: ['ip', 'ipv4', 'ipv6', 'mac'],
+    },
+    {
+        kind: 'personal', what: 'a postal address or location',
+        substrings: ['address', 'street', 'postalcode', 'postcode', 'zipcode', 'latitude', 'longitude', 'geolocation',
+            'coordinates', 'birthplace'],
+        tokens: ['addr', 'zip', 'postal', 'city', 'town', 'county', 'province', 'suburb', 'country', 'lat', 'lng', 'lon',
+            'geo', 'location'],
+    },
+    {
+        kind: 'personal', what: "a person's name or handle",
+        substrings: ['firstname', 'lastname', 'fullname', 'givenname', 'familyname', 'middlename', 'maidenname',
+            'displayname', 'username', 'nickname', 'surname', 'forename'],
+        tokens: ['name', 'names', 'fname', 'lname', 'login', 'handle', 'initials', 'signature'],
+    },
+    {
+        kind: 'free-text', what: 'free text',
+        substrings: ['description', 'biography', 'instruction', 'transcript'],
+        tokens: ['note', 'notes', 'comment', 'comments', 'desc', 'message', 'messages', 'msg', 'body', 'content', 'text',
+            'bio', 'about', 'summary', 'remark', 'remarks', 'feedback', 'memo', 'subject', 'title', 'caption', 'review',
+            'reply', 'answer', 'question', 'prompt', 'story'],
+    },
+];
+/** A name that starts like `is_`/`has_` or ends like `_at`/`_count`/`_type`
+ * describes the personal field rather than holding it (`email_verified_at`,
+ * `phone_type`, `has_notes`). The value-shape tests still run on these. */
+const DESCRIBING_PREFIXES = ['is', 'has', 'can', 'should', 'allow', 'allows', 'uses', 'needs', 'wants'];
+const DESCRIBING_SUFFIXES = ['at', 'verified', 'confirmed', 'count', 'type', 'kind', 'status', 'enabled', 'disabled',
+    'expires', 'expiry', 'changed', 'updated', 'sent', 'version', 'provider', 'template', 'length', 'size', 'format',
+    'locale', 'language', 'consent', 'optin'];
+export function classifyColumnName(name) {
+    const tokens = columnNameTokens(name);
+    if (tokens.length === 0)
+        return null;
+    if (tokens.length > 1 && (DESCRIBING_PREFIXES.includes(tokens[0]) || DESCRIBING_SUFFIXES.includes(tokens[tokens.length - 1]))) {
+        return null;
+    }
+    const compact = tokens.join('');
+    for (const rule of PERSONAL_NAME_RULES) {
+        const substring = rule.substrings.find(word => compact.includes(word));
+        const token = rule.tokens.find(word => tokens.includes(word));
+        const matched = substring ?? token;
+        if (matched !== undefined) {
+            return { kind: rule.kind, reason: `column name suggests ${rule.what} (matches "${matched}")` };
+        }
+    }
+    return null;
+}
+export function classifyColumnType(family, dataType) {
+    if (family === 'network') {
+        return { kind: 'personal', reason: `column type ${dataType} holds network or device addresses` };
+    }
+    return null;
+}
+export const VALUE_SHAPE_TESTS = [
+    {
+        id: 'email', threshold: 0.05, what: 'email addresses',
+        predicate: v => `${v} ~ '^[^@[:space:]]+@[^@[:space:]]+\\.[^@[:space:]]+$'`,
+    },
+    {
+        id: 'phone', threshold: 0.2, what: 'phone numbers',
+        predicate: v => `(${v} ~ '^\\+?[0-9 ().-]{7,20}$' AND ${v} ~ '([0-9][^0-9]*){7}')`,
+    },
+    {
+        id: 'network', threshold: 0.05, what: 'IP addresses',
+        predicate: v => `(${v} ~ '^[0-9]{1,3}(\\.[0-9]{1,3}){3}$' OR ${v} ~ '^[0-9A-Fa-f]{0,4}(:[0-9A-Fa-f]{0,4}){2,7}$')`,
+    },
+    {
+        id: 'secret', threshold: 0.3, what: 'secrets, tokens or password hashes',
+        predicate: v => `((${v} ~ '^[A-Za-z0-9+/_=.$:-]{20,}$' AND ${v} ~ '[A-Z]' AND ${v} ~ '[a-z]' AND ${v} ~ '[0-9]')`
+            + ` OR ${v} ~ '^[0-9A-Fa-f]{32,}$' OR ${v} ~ '^\\$(2[aby]|argon2|scrypt|pbkdf2)')`,
+    },
+    {
+        id: 'government-id', threshold: 0.05, what: 'government identifiers (SSN shape)',
+        predicate: v => `${v} ~ '^[0-9]{3}-[0-9]{2}-[0-9]{4}$'`,
+    },
+    {
+        id: 'card', threshold: 0.05, what: 'payment card numbers',
+        predicate: v => `(${v} ~ '^[0-9][0-9 -]{11,21}[0-9]$' AND ${v} ~ '([0-9][^0-9]*){13}')`,
+    },
+    {
+        id: 'person-name', threshold: 0.5, what: "people's names (two or more capitalised words)",
+        predicate: v => `${v} ~ '^[[:upper:]][[:lower:]]+([ ''-][[:upper:]][[:lower:]]+)+$'`,
+    },
+];
+/**
+ * The shapes above except person-name (common category values such as "New
+ * York" have it) as single-value tests (the same patterns, in JavaScript),
+ * for values about to leave. A column is personal only when a threshold share
+ * of its values has a shape, so a category column can still hold a common
+ * value of one of these shapes ("2031-07-19" is phone-shaped, a shared API
+ * token is secret-shaped). Such a value never leaves, whatever its column's
+ * type (see `keptValueProblem`): the profiler drops it from the kept values
+ * and withholds any JSON key path with a key of such a shape, and the upload
+ * server refuses a profile that keeps one.
+ * Values are tested trimmed. Both sides use this one list.
+ */
+export const KEPT_VALUE_PERSONAL_SHAPES = [
+    { id: 'email', what: 'an email address', test: value => /^[^@\s]+@[^@\s]+\.[^@\s]+$/u.test(value) },
+    { id: 'phone', what: 'a phone number', test: value => /^\+?[0-9 ().-]{7,20}$/u.test(value) && /([0-9][^0-9]*){7}/u.test(value) },
+    {
+        id: 'network', what: 'an IP address',
+        test: value => /^[0-9]{1,3}(\.[0-9]{1,3}){3}$/u.test(value) || /^[0-9A-Fa-f]{0,4}(:[0-9A-Fa-f]{0,4}){2,7}$/u.test(value),
+    },
+    { id: 'government-id', what: 'a government identifier', test: value => /^[0-9]{3}-[0-9]{2}-[0-9]{4}$/u.test(value) },
+    { id: 'card', what: 'a payment card number', test: value => /^[0-9][0-9 -]{11,21}[0-9]$/u.test(value) && /([0-9][^0-9]*){13}/u.test(value) },
+    {
+        id: 'secret', what: 'a secret, token or password hash',
+        test: value => (/^[A-Za-z0-9+/_=.$:-]{20,}$/u.test(value) && /[A-Z]/u.test(value) && /[a-z]/u.test(value) && /[0-9]/u.test(value))
+            || /^[0-9A-Fa-f]{32,}$/u.test(value) || /^\$(2[aby]|argon2|scrypt|pbkdf2)/u.test(value),
+    },
+];
+/** What personal shape a value about to leave has, or null. */
+export function keptValuePersonalShape(value) {
+    const trimmed = value.trim();
+    return KEPT_VALUE_PERSONAL_SHAPES.find(shape => shape.test(trimmed))?.what ?? null;
+}
+/**
+ * The personal shape a JSON key path shows, if any: its whole text, or any run
+ * of consecutive dot-separated segments (with `[]` element markers dropped).
+ * A key may itself hold a dot, so the runs cover every key the path can have
+ * been made of, and also keys that only read as personal together
+ * (`stats.2024.01.15` holds `2024.01.15`, a phone number's shape). The
+ * profiler keeps no path this names, and the upload check refuses one, with
+ * this same function, so a freshly written profile always passes it.
+ */
+export function jsonPathPersonalShape(path) {
+    const whole = keptValuePersonalShape(path);
+    if (whole)
+        return whole;
+    const segments = path.split('.').map(segment => segment.replace(/(\[\])+$/u, ''));
+    for (let start = 0; start < segments.length; start += 1) {
+        for (let end = start + 1; end <= segments.length; end += 1) {
+            const shape = keptValuePersonalShape(segments.slice(start, end).join('.'));
+            if (shape)
+                return shape;
+        }
+    }
+    return null;
+}
+const DATE_TEXT = '[0-9]{4,}-[0-9]{2}-[0-9]{2}';
+const TIME_TEXT = '[0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]+)?';
+const ZONE_TEXT = '[+-][0-9]{2}(:[0-9]{2}){0,2}';
+/**
+ * The declared types (as format_type prints them) whose data values may
+ * leave, and the text Postgres gives such a value (`value::text`, ISO
+ * DateStyle). A value that does not fit its column's declared type is not a
+ * value of that column (a file can say `integer` over an email address), and
+ * a data value of any other type (arrays, domains, enums, extension types)
+ * never leaves: enum labels leave as schema values.
+ */
+export const KEPT_VALUE_TYPES = [
+    { dataType: /^(text|character varying|character|citext|name)(\([0-9]+\))?$/u, value: /^[\s\S]*$/u },
+    { dataType: /^(smallint|integer|bigint)$/u, value: /^-?[0-9]+$/u },
+    {
+        dataType: /^(numeric(\([0-9]+(,[0-9]+)?\))?|real|double precision)$/u,
+        value: /^(-?([0-9]+(\.[0-9]*)?|\.[0-9]+)([eE][-+]?[0-9]+)?|NaN|-?Infinity)$/u,
+    },
+    { dataType: /^boolean$/u, value: /^(true|false)$/u },
+    { dataType: /^date$/u, value: new RegExp(`^(${DATE_TEXT}( BC)?|-?infinity)$`, 'u') },
+    { dataType: /^time(\([0-6]\))? without time zone$/u, value: new RegExp(`^${TIME_TEXT}$`, 'u') },
+    { dataType: /^time(\([0-6]\))? with time zone$/u, value: new RegExp(`^${TIME_TEXT}${ZONE_TEXT}$`, 'u') },
+    {
+        dataType: /^timestamp(\([0-6]\))? without time zone$/u,
+        value: new RegExp(`^(${DATE_TEXT} ${TIME_TEXT}( BC)?|-?infinity)$`, 'u'),
+    },
+    {
+        dataType: /^timestamp(\([0-6]\))? with time zone$/u,
+        value: new RegExp(`^(${DATE_TEXT} ${TIME_TEXT}${ZONE_TEXT}( BC)?|-?infinity)$`, 'u'),
+    },
+];
+/**
+ * Why a data value about to leave from a column of the declared `dataType`
+ * may not: it has a personal shape (whatever the type: a number of seven or
+ * more digits is phone-shaped), its column's type is not one whose data
+ * values may leave, or it does not fit that type. Null when it may leave.
+ * The profiler drops such values and the upload server refuses them, by this
+ * one rule. The reason never quotes the value.
+ */
+export function keptValueProblem(dataType, value) {
+    const shape = keptValuePersonalShape(value);
+    if (shape !== null)
+        return `looks like ${shape}`;
+    const type = KEPT_VALUE_TYPES.find(candidate => candidate.dataType.test(dataType));
+    if (type === undefined)
+        return `is data of type ${dataType}, whose values never leave`;
+    if (!type.value.test(value))
+        return `does not fit its column's type (${dataType})`;
+    return null;
+}
+/** Free text: many words per value. Measured as the average number of words
+ * (spaces + 1) and the share of values with four or more words. */
+export const FREE_TEXT_AVERAGE_WORDS = 3.5;
+export const FREE_TEXT_FOUR_WORD_SHARE = 0.25;
+export function averageWordsSql(value) {
+    return `avg(length(btrim(${value})) - length(replace(btrim(${value}), ' ', '')) + 1)`;
+}
+export function fourWordsPredicate(value) {
+    return `${value} ~ '[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]'`;
+}
+export function classifyValueShapes(measured) {
+    if (measured.nonNull === 0)
+        return null;
+    for (const test of VALUE_SHAPE_TESTS) {
+        const share = measured.matches[test.id] / measured.nonNull;
+        if (share >= test.threshold) {
+            return { kind: 'personal', reason: `${formatShare(share)} of values look like ${test.what}` };
+        }
+    }
+    const fourWordShare = measured.fourWordValues / measured.nonNull;
+    if ((measured.averageWords !== null && measured.averageWords >= FREE_TEXT_AVERAGE_WORDS)
+        || fourWordShare >= FREE_TEXT_FOUR_WORD_SHARE) {
+        const words = measured.averageWords === null ? 'unknown' : measured.averageWords.toFixed(1);
+        return {
+            kind: 'free-text',
+            reason: `values look like free text (${words} words on average; ${formatShare(fourWordShare)} have four or more words)`,
+        };
+    }
+    return null;
+}
+/** A share as a percentage that never rounds a partial share to 0% or 100%. */
+export function formatShare(share) {
+    const percent = share * 100;
+    if (percent <= 0 || percent >= 100)
+        return `${Math.round(percent)}%`;
+    if (percent < 0.1)
+        return '<0.1%';
+    if (percent > 99.9)
+        return '>99.9%';
+    return `${percent.toFixed(percent >= 10 && percent <= 90 ? 0 : 1)}%`;
+}
+/* ------------------------------------------------------------ shapes */
+/**
+ * Character-class shape of a text value, computed in SQL: uppercase letter
+ * runs become `A`, lowercase and uncased letter runs `a`, digit runs `9`,
+ * whitespace runs one space; ASCII punctuation is kept; anything else becomes
+ * `?`; then repeated characters collapse. `Zoë O'Neil-Smith 42` becomes
+ * `Aa? A'Aa-Aa 9` (or `Aa A'Aa-Aa 9` where the database locale knows ë).
+ */
+export function shapeSql(value) {
+    const classes = `regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(${value}, `
+        + `'[[:upper:]]+', chr(1), 'g'), '[[:lower:]]+', chr(2), 'g'), '[[:alpha:]]+', chr(2), 'g'), `
+        + `'[[:digit:]]+', chr(3), 'g'), '[[:space:]]+', ' ', 'g')`;
+    const named = `translate(${classes}, chr(1) || chr(2) || chr(3), 'Aa9')`;
+    const bounded = `regexp_replace(${named}, '[^Aa9 \\x21-\\x2f\\x3a-\\x40\\x5b-\\x60\\x7b-\\x7e]', '?', 'g')`;
+    return `regexp_replace(${bounded}, '(.)\\1+', '\\1', 'g')`;
+}
+const IDENTIFIER_NAME = /(^id$|_id$|Id$|_uuid$|_guid$)/u;
+export function classifyColumnFacts(facts) {
+    const personal = classifyColumnType(facts.family, facts.dataType)
+        ?? classifyColumnName(facts.name)
+        ?? (facts.valueShapes === null ? null : classifyValueShapes(facts.valueShapes));
+    if (personal !== null)
+        return { kind: personal.kind, personalReason: personal.reason };
+    if (facts.family === 'boolean')
+        return { kind: 'boolean', personalReason: null };
+    if (facts.family === 'json')
+        return { kind: 'json', personalReason: null };
+    if (facts.family === 'binary')
+        return { kind: 'binary', personalReason: null };
+    if (facts.family === 'uuid' || facts.isKey || IDENTIFIER_NAME.test(facts.name)) {
+        return { kind: 'identifier', personalReason: null };
+    }
+    if (facts.family === 'time')
+        return { kind: 'time', personalReason: null };
+    if (facts.schemaValues !== null)
+        return { kind: 'category', personalReason: null };
+    if (facts.family === 'text' || facts.family === 'number' || facts.family === 'enum') {
+        return { kind: 'distinct-decides', otherwise: facts.family === 'number' ? 'number' : 'other' };
+    }
+    return { kind: 'other', personalReason: null };
+}
+/**
+ * Settle a column by its distinct count. `exactDistinct` must be the exact
+ * number of distinct non-null values in the whole table, never a planner or
+ * sample estimate: an underestimate would let a high-cardinality column keep
+ * values. A count known only to exceed `categoryMaxDistinct` may be passed as
+ * any number above it.
+ */
+export function decideByDistinct(verdict, exactDistinct, categoryMaxDistinct) {
+    if (exactDistinct >= 1 && exactDistinct <= categoryMaxDistinct)
+        return { kind: 'category', personalReason: null };
+    return { kind: verdict.otherwise, personalReason: null };
+}
+export function classifyColumn(input) {
+    const verdict = classifyColumnFacts(input);
+    return verdict.kind === 'distinct-decides' ? decideByDistinct(verdict, input.exactDistinct, input.categoryMaxDistinct) : verdict;
+}
+/* ------------------------------------------------------------ CHECK lists */
+/**
+ * The allowed values of a single-column CHECK constraint, when the
+ * constraint is exactly a value list (`col IN (...)`, printed by Postgres as
+ * `col = ANY (ARRAY[...])`, or an OR of `col = literal`). Any other shape
+ * (ranges, functions, AND-ed conditions, NOT IN) is not a value list and
+ * returns null. Only the definition text is read; no data.
+ */
+export function checkConstraintValues(definition, column) {
+    const body = definition.trim().match(/^CHECK \((.*)\)$/su)?.[1];
+    if (body === undefined)
+        return null;
+    const columnRef = `(?:\\(*${escapeRegExp(quoteIdentifierForMatch(column))}\\)*(?:::[a-z ]+(?:\\[\\])?)?)`;
+    const literal = `(?:'(?:[^']|'')*'(?:::[a-z ]+)?|-?[0-9]+(?:\\.[0-9]+)?)`;
+    const anyForm = new RegExp(`^\\(*${columnRef} = ANY \\(+ARRAY\\[(${literal}(?:, ${literal})*)\\](?:\\)*::[a-z ]+\\[\\])?\\)+$`, 'u');
+    const anyMatch = body.match(anyForm);
+    if (anyMatch)
+        return literalList(anyMatch[1]);
+    const equality = `\\(*${columnRef} = (${literal})\\)*`;
+    const terms = splitTopLevelOr(stripOuterParens(body));
+    if (terms === null)
+        return null;
+    const values = [];
+    for (const term of terms) {
+        const match = term.match(new RegExp(`^${equality}$`, 'u'));
+        if (!match)
+            return null;
+        values.push(...literalList(match[1]));
+    }
+    return values.length > 0 ? values : null;
+}
+function quoteIdentifierForMatch(column) {
+    return /^[a-z_][a-z0-9_$]*$/u.test(column) ? column : `"${column.replace(/"/gu, '""')}"`;
+}
+function escapeRegExp(value) {
+    return value.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
+}
+function literalList(items) {
+    const values = [];
+    const pattern = /'((?:[^']|'')*)'(?:::[a-z ]+)?|(-?[0-9]+(?:\.[0-9]+)?)/gu;
+    for (const match of items.matchAll(pattern)) {
+        values.push(match[1] !== undefined ? match[1].replace(/''/gu, "'") : match[2]);
+    }
+    return values;
+}
+function stripOuterParens(value) {
+    let current = value.trim();
+    while (current.startsWith('(') && current.endsWith(')') && closingParen(current, 0) === current.length - 1) {
+        current = current.slice(1, -1).trim();
+    }
+    return current;
+}
+function closingParen(value, open) {
+    let depth = 0;
+    let quoted = false;
+    for (let index = open; index < value.length; index += 1) {
+        const char = value[index];
+        if (char === "'")
+            quoted = !quoted;
+        if (quoted)
+            continue;
+        if (char === '(')
+            depth += 1;
+        if (char === ')') {
+            depth -= 1;
+            if (depth === 0)
+                return index;
+        }
+    }
+    return -1;
+}
+function splitTopLevelOr(value) {
+    const terms = [];
+    let depth = 0;
+    let quoted = false;
+    let start = 0;
+    for (let index = 0; index < value.length; index += 1) {
+        const char = value[index];
+        if (char === "'")
+            quoted = !quoted;
+        if (quoted)
+            continue;
+        if (char === '(')
+            depth += 1;
+        if (char === ')')
+            depth -= 1;
+        if (depth < 0)
+            return null;
+        if (depth === 0 && value.startsWith(' OR ', index)) {
+            terms.push(value.slice(start, index).trim());
+            start = index + 4;
+        }
+        if (depth === 0 && value.startsWith(' AND ', index))
+            return null;
+    }
+    terms.push(value.slice(start).trim());
+    return terms;
+}

package/dist/commands/db-profile-show.js

@@ -0,0 +1,218 @@
+/**
+ * `haystack db profile show <file>` — a plain-English review of exactly what
+ * `haystack db profile upload` would send. Every string the file carries
+ * (column names and types, category values, shapes, JSON key paths, odd-row
+ * descriptions, index definitions, settings) is printed; the strict parser
+ * refuses a file with fields this review does not print.
+ */
+import { readFileSync } from 'node:fs';
+import { parseDatabaseProfile, } from './db-profile-contract.js';
+import { describeReach, ownerReach } from './db-profile-owners.js';
+import { formatShare as pct } from './db-profile-privacy.js';
+export function readDatabaseProfileFile(file) {
+    let raw;
+    try {
+        raw = readFileSync(file, 'utf8');
+    }
+    catch (error) {
+        throw new Error(`Could not read ${file}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (error) {
+        throw new Error(`${file} is not JSON: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    return parseDatabaseProfile(parsed, file);
+}
+const PERSONAL_KINDS = new Set(['personal', 'free-text']);
+function count(value) {
+    return value.toLocaleString('en-US');
+}
+function bytes(value) {
+    const units = ['B', 'kB', 'MB', 'GB', 'TB'];
+    let scaled = value;
+    let unit = 0;
+    while (scaled >= 1024 && unit < units.length - 1) {
+        scaled /= 1024;
+        unit += 1;
+    }
+    return `${unit === 0 ? scaled.toFixed(0) : scaled.toFixed(1)} ${units[unit]}`;
+}
+function quote(value) {
+    return JSON.stringify(value);
+}
+function quantileText(quantiles, format) {
+    return [quantiles.p5, quantiles.p25, quantiles.p50, quantiles.p75, quantiles.p95].map(format).join(' / ');
+}
+function numberFormat(column) {
+    const type = column.dataType.toLowerCase();
+    if (type === 'date')
+        return value => new Date(value * 1000).toISOString().slice(0, 10);
+    if (type.startsWith('timestamp'))
+        return value => new Date(value * 1000).toISOString().replace('.000Z', 'Z');
+    if (type.startsWith('time') || type.startsWith('interval'))
+        return value => `${value}s`;
+    return value => String(value);
+}
+function kindLabel(kind) {
+    return kind === 'free-text' ? 'free text' : kind;
+}
+function renderColumn(column, rules, fullScan, ownersCountable) {
+    const lines = [];
+    const categories = column.categories ?? [];
+    let headline;
+    if (PERSONAL_KINDS.has(column.kind)) {
+        headline = `PERSONAL (${kindLabel(column.kind)}), values never leave: ${column.personalReason}`;
+    }
+    else if (column.kind === 'category' && categories.length > 0) {
+        const sources = new Set(categories.map(entry => entry.source));
+        headline = sources.has('schema')
+            ? 'category, KEEPS VALUES declared by the schema (enum type or CHECK list), with their share of rows'
+            : `category, KEEPS VALUES that are each shared by at least ${rules.categoryMinOwners} owners`;
+    }
+    else if (column.kind === 'category') {
+        headline = `category, keeps no values (none is shared by at least ${rules.categoryMinOwners} owners without looking like personal data, or owners cannot be counted for this table)`;
+    }
+    else {
+        headline = `${kindLabel(column.kind)}, numbers only`;
+    }
+    lines.push(`    ${column.name} (${column.dataType}${column.notNull ? ', not null' : ''}): ${headline}`);
+    if (categories.length > 0) {
+        const kept = categories.reduce((sum, entry) => sum + entry.fraction, 0);
+        lines.push(`        values: ${categories.map(entry => `${quote(entry.value)} ${pct(entry.fraction)}`).join(', ')}`);
+        if (categories.every(entry => entry.source === 'data') && kept < 0.9995) {
+            lines.push(`        withheld: ${pct(1 - kept)} of non-null values (values shared by too few owners, or shaped like personal data)`);
+        }
+    }
+    // A table read in full has exact distinct counts; a sampled one mostly estimates.
+    lines.push(`        ${pct(column.nullFraction)} null, ${fullScan ? '' : 'about '}${count(column.distinctEstimate)} distinct, ${column.avgWidthBytes} bytes on average`);
+    if (column.lengthQuantiles !== null) {
+        lines.push(`        length in bytes, 5th/25th/50th/75th/95th percentile: ${quantileText(column.lengthQuantiles, value => String(value))}`);
+    }
+    if (column.numberQuantiles !== null) {
+        lines.push(`        values, 5th/25th/50th/75th/95th percentile: ${quantileText(column.numberQuantiles, numberFormat(column))}`);
+    }
+    if (column.booleanTrueFraction !== null) {
+        lines.push(`        true in ${pct(column.booleanTrueFraction)} of non-null values`);
+    }
+    else if (column.kind === 'boolean') {
+        lines.push(ownersCountable
+            ? `        share of true not sent: the column has no values, or fewer than ${rules.categoryMinOwners} owners have one`
+            : '        share of true not sent: owners cannot be counted for this table');
+    }
+    if (column.shapes.length > 0) {
+        lines.push(`        shapes: ${column.shapes.map(entry => `${quote(entry.shape)} ${pct(entry.fraction)}`).join(', ')}`);
+    }
+    for (const key of column.jsonKeys) {
+        lines.push(`        JSON key ${quote(key.path)}: ${key.types.join(' or ')}, in ${pct(key.presentFraction)} of rows`);
+    }
+    for (const oddity of column.oddities) {
+        const at = oddity.jsonPath === null ? '' : ` at JSON key ${quote(oddity.jsonPath)}`;
+        const types = oddity.types === null ? '' : `, expected ${oddity.types.expected}, found ${oddity.types.found.join(' or ')}`;
+        lines.push(`        odd rows (${oddity.kind}, ${count(oddity.rowCount)}${at}${types}): ${oddity.description}`);
+    }
+    return lines;
+}
+function tableEdges(profile) {
+    const profiled = new Set(profile.tables.map(table => `${table.schema}.${table.name}`));
+    return profile.tables.flatMap(table => table.foreignKeys
+        .filter(key => profiled.has(`${key.references.schema}.${key.references.table}`))
+        .map(key => ({
+        name: key.name,
+        from: { schema: table.schema, table: table.name },
+        columns: key.columns,
+        to: { schema: key.references.schema, table: key.references.table },
+        referencedColumns: key.references.columns,
+    })));
+}
+function renderTable(table, profile, edges) {
+    const owner = profile.rules.ownerCounting.kind === 'owner-table'
+        ? { schema: profile.rules.ownerCounting.schema, table: profile.rules.ownerCounting.table }
+        : null;
+    const rowSource = table.rowCountSource === 'exact' ? 'counted exactly' : "the planner's estimate";
+    const dead = table.deadRows.status === 'measured'
+        ? `${pct(table.deadRows.fraction)} dead rows`
+        : `dead rows not measured: ${table.deadRows.reason}`;
+    const fullScan = table.sampling.status === 'full-scan';
+    const reach = ownerReach({ schema: table.schema, table: table.name }, owner, edges);
+    const reading = table.sampling.status === 'full-scan'
+        ? 'every row'
+        : `a ${table.sampling.percent}% TABLESAMPLE SYSTEM sample (seed ${table.sampling.seed}), so row counts in odd-row findings are counts in that sample`;
+    const lines = [
+        '',
+        `TABLE ${table.schema}.${table.name}`,
+        `  ${count(table.rowCount)} rows (${rowSource}); table ${bytes(table.tableBytes)}, indexes ${bytes(table.indexBytes)}, TOAST ${bytes(table.toastBytes)}`,
+        `  ${dead}`,
+        `  Measured from ${reading}`,
+        `  Owners: ${describeReach(reach, owner)}`,
+    ];
+    const keeping = table.columns.filter(column => (column.categories ?? []).length > 0);
+    const personal = table.columns.filter(column => PERSONAL_KINDS.has(column.kind));
+    const rest = table.columns.filter(column => !keeping.includes(column) && !personal.includes(column));
+    lines.push(`  Columns that keep values (${keeping.length}):`);
+    lines.push(...(keeping.length === 0 ? ['    none'] : keeping.flatMap(column => renderColumn(column, profile.rules, fullScan, reach.kind !== 'unreachable'))));
+    lines.push(`  Personal columns, whose values never leave (${personal.length}):`);
+    lines.push(...(personal.length === 0 ? ['    none'] : personal.flatMap(column => renderColumn(column, profile.rules, fullScan, reach.kind !== 'unreachable'))));
+    lines.push(`  Other columns, numbers and shapes only (${rest.length}):`);
+    lines.push(...(rest.length === 0 ? ['    none'] : rest.flatMap(column => renderColumn(column, profile.rules, fullScan, reach.kind !== 'unreachable'))));
+    lines.push(`  Foreign keys (${table.foreignKeys.length}):`);
+    for (const key of table.foreignKeys) {
+        lines.push(`    ${key.name}: ${key.columns.join(', ')} -> ${key.references.schema}.${key.references.table}(${key.references.columns.join(', ')}); `
+            + `children per parent, 5th-95th percentile: ${quantileText(key.childrenPerParent, value => String(value))}; ${pct(key.orphanFraction)} orphans`);
+    }
+    lines.push(`  Index definitions, written by your developers, every literal value replaced with ? (${table.indexes.length}):`);
+    for (const index of table.indexes)
+        lines.push(`    ${index.name}: ${index.definition}`);
+    return lines;
+}
+export function renderDatabaseProfile(profile, file) {
+    const rules = profile.rules;
+    const columns = profile.tables.flatMap(table => table.columns);
+    const keeping = columns.filter(column => (column.categories ?? []).length > 0);
+    const fromSchema = keeping.filter(column => (column.categories ?? []).some(entry => entry.source === 'schema')).length;
+    const personal = columns.filter(column => PERSONAL_KINDS.has(column.kind)).length;
+    const oddRows = columns.reduce((sum, column) => sum + column.oddities.length, 0);
+    const ownerLine = rules.ownerCounting.kind === 'owner-table'
+        ? `Owners are distinct rows of ${rules.ownerCounting.schema}.${rules.ownerCounting.table}, reached from each table by following its foreign keys.`
+        : `Owners are rows: ${rules.ownerCounting.reason}`;
+    const edges = tableEdges(profile);
+    return [
+        `Haystack database profile: ${file}`,
+        `Made by ${profile.profilerVersion} at ${profile.createdAt} from ${profile.engine} ${profile.engineVersion} (format ${profile.version}).`,
+        '',
+        'WHAT THIS FILE SENDS',
+        'This file is everything `haystack db profile upload` sends, and it holds no rows. A column\'s',
+        'values appear below only when all three rules hold:',
+        `  1. the column has at most ${rules.categoryMaxDistinct} distinct values (a category column);`,
+        '  2. the column is not personal: names, emails, phones, addresses, free text, secrets,',
+        '     network addresses, dates of birth, government IDs. When in doubt it is personal;',
+        `  3. each value is shared by at least ${rules.categoryMinOwners} owners.`,
+        'Values declared by the schema itself (enum labels, CHECK value lists) need no data and are',
+        'listed with their share of rows.',
+        ownerLine,
+        'Everything else is numbers (row counts, sizes, null shares, distinct counts, average widths,',
+        'the share of true in yes/no columns, and 5th to 95th percentiles, never the minimum or maximum;',
+        `both only where at least ${rules.categoryMinOwners} owners have a value), character shapes and JSON key`,
+        `paths each shared by at least ${rules.categoryMinOwners} owners, and descriptions of odd rows that never quote a value.`,
+        'Index definitions are sent with every literal value replaced with ?.',
+        'In shapes, A is a run of capital letters, a a run of small letters, 9 a run of digits, and',
+        '? any other character; punctuation is kept.',
+        '',
+        'SUMMARY',
+        `  ${profile.tables.length} tables, ${columns.length} columns.`,
+        `  ${keeping.length} columns keep values (${fromSchema} from the schema, ${keeping.length - fromSchema} from the data).`,
+        `  ${personal} columns are personal and keep no values.`,
+        `  ${oddRows} odd-row findings, counts only.`,
+        '',
+        'SERVER SETTINGS (planner and storage only)',
+        ...profile.settings.map(setting => `  ${setting.name} = ${setting.value}`),
+        ...profile.tables.flatMap(table => renderTable(table, profile, edges)),
+        '',
+    ].join('\n');
+}
+export async function databaseProfileShowCommand(file) {
+    const profile = readDatabaseProfileFile(file);
+    process.stdout.write(renderDatabaseProfile(profile, file));
+}