@haystackeditor/cli 0.17.1 → 0.19.0

package/dist/commands/db-profile-contract.js

@@ -0,0 +1,419 @@
+/** FIXED CONTRACT — mirror of the PII-free database profile shapes for the
+ * published CLI, whose build cannot import repository files outside `src`.
+ * The authority is `infra/lambda/haystack-design-verifier-shared/onboarding_parity_contracts.ts`
+ * (frozen at 886d044106ebe41ba778b8392d927cd47cef826c, then format v2: the
+ * oddities' `jsonPath` and `types`, added to both together): `DatabaseProfileV1`,
+ * `TableProfileV1`, `ColumnProfileV1`, `ColumnKindV1`, `QuantilesV1`,
+ * `DatabaseProfileRulesV1`, `DatabaseProfileUploadRequestV1` and
+ * `DatabaseProfileUploadV1`, plus `ArtifactReference` from
+ * `fleet_bake_contracts.ts`. The shapes are restated field for field; the
+ * comments that carry the privacy rules are restated where the CLI enforces
+ * them.
+ *
+ * The parser below is strict: a field the format does not define is refused,
+ * because `haystack db profile show` renders exactly the defined fields and a
+ * profile must never carry something the customer was not shown. It also
+ * refuses an index definition that still holds a literal.
+ */
+import { redactSqlLiterals, SqlLiteralError } from './db-profile-sql-literals.js';
+export const COLUMN_KINDS = [
+    'category', 'personal', 'free-text', 'identifier', 'number',
+    'time', 'boolean', 'json', 'binary', 'other',
+];
+export const ODDITY_KINDS = [
+    'type-mismatch', 'oversize', 'invalid-encoding', 'unexpected-null', 'shape-outlier', 'missing-json-key',
+];
+/** JSON value types, as jsonb_typeof names them. */
+export const JSON_TYPE_NAMES = ['object', 'array', 'string', 'number', 'boolean', 'null'];
+/** Type names an oddity's `types` may hold: the JSON types, plus 'iso-date'
+ * and 'text' for text that is mostly numbers or ISO dates, and 'finite' and
+ * 'nan-or-infinity' for a number column holding NaN or infinity. */
+export const ODDITY_TYPE_NAMES = [...JSON_TYPE_NAMES, 'iso-date', 'text', 'finite', 'nan-or-infinity'];
+/** v2 added the oddities' `jsonPath` and `types`. */
+export const DATABASE_PROFILE_VERSION = 'dv-database-profile-v2';
+/** No published CLI wrote v1, so a v1 file is refused rather than migrated. */
+export const OLDER_PROFILE_MESSAGE = 'this profile was written by an older haystack CLI; run haystack db profile again';
+export const DATABASE_PROFILE_UPLOAD_REQUEST_VERSION = 'dv-database-profile-upload-request-v1';
+export const DATABASE_PROFILE_UPLOAD_VERSION = 'dv-database-profile-upload-v1';
+export class DatabaseProfileFormatError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'DatabaseProfileFormatError';
+    }
+}
+/* ------------------------------------------------------------ strict parse */
+function fail(message) {
+    throw new DatabaseProfileFormatError(message);
+}
+function record(value, label, keys) {
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        fail(`${label} must be an object.`);
+    const fields = value;
+    for (const key of Object.keys(fields)) {
+        if (!keys.includes(key)) {
+            fail(`${label} has a field the profile format does not define ("${key}"); remove it so nothing is sent that \`haystack db profile show\` does not display.`);
+        }
+    }
+    for (const key of keys) {
+        if (!(key in fields))
+            fail(`${label} is missing "${key}".`);
+    }
+    return fields;
+}
+function text(value, label) {
+    if (typeof value !== 'string')
+        fail(`${label} must be a string.`);
+    return value;
+}
+function nonEmptyText(value, label) {
+    const parsed = text(value, label);
+    if (parsed.length === 0)
+        fail(`${label} must not be empty.`);
+    return parsed;
+}
+function bool(value, label) {
+    if (typeof value !== 'boolean')
+        fail(`${label} must be true or false.`);
+    return value;
+}
+function finite(value, label) {
+    if (typeof value !== 'number' || !Number.isFinite(value))
+        fail(`${label} must be a finite number.`);
+    return value;
+}
+function count(value, label) {
+    const parsed = finite(value, label);
+    if (!Number.isInteger(parsed) || parsed < 0)
+        fail(`${label} must be a non-negative integer.`);
+    return parsed;
+}
+function positiveInteger(value, label) {
+    const parsed = count(value, label);
+    if (parsed < 1)
+        fail(`${label} must be at least 1.`);
+    return parsed;
+}
+function fraction(value, label) {
+    const parsed = finite(value, label);
+    if (parsed < 0 || parsed > 1)
+        fail(`${label} must be between 0 and 1.`);
+    return parsed;
+}
+function nonNegative(value, label) {
+    const parsed = finite(value, label);
+    if (parsed < 0)
+        fail(`${label} must not be negative.`);
+    return parsed;
+}
+function list(value, label) {
+    if (!Array.isArray(value))
+        fail(`${label} must be an array.`);
+    return value;
+}
+function oneOf(value, label, allowed) {
+    if (typeof value !== 'string' || !allowed.includes(value)) {
+        fail(`${label} must be one of ${allowed.join(', ')}.`);
+    }
+    return value;
+}
+function isoTime(value, label) {
+    const parsed = nonEmptyText(value, label);
+    if (Number.isNaN(Date.parse(parsed)))
+        fail(`${label} must be an ISO-8601 time.`);
+    return parsed;
+}
+export function parseQuantiles(value, label) {
+    // Exactly p5..p95: a profile that carries min, max or any other extreme is refused.
+    const fields = record(value, label, ['p5', 'p25', 'p50', 'p75', 'p95']);
+    const quantiles = {
+        p5: finite(fields.p5, `${label}.p5`),
+        p25: finite(fields.p25, `${label}.p25`),
+        p50: finite(fields.p50, `${label}.p50`),
+        p75: finite(fields.p75, `${label}.p75`),
+        p95: finite(fields.p95, `${label}.p95`),
+    };
+    const ordered = [quantiles.p5, quantiles.p25, quantiles.p50, quantiles.p75, quantiles.p95];
+    for (let index = 1; index < ordered.length; index += 1) {
+        if (ordered[index] < ordered[index - 1])
+            fail(`${label} must be non-decreasing from p5 to p95.`);
+    }
+    return quantiles;
+}
+function parseRules(value, label) {
+    const fields = record(value, label, ['categoryMaxDistinct', 'categoryMinOwners', 'ownerCounting']);
+    const owner = fields.ownerCounting;
+    if (!owner || typeof owner !== 'object' || Array.isArray(owner))
+        fail(`${label}.ownerCounting must be an object.`);
+    const kind = owner.kind;
+    let ownerCounting;
+    if (kind === 'owner-table') {
+        const table = record(owner, `${label}.ownerCounting`, ['kind', 'schema', 'table']);
+        ownerCounting = {
+            kind: 'owner-table',
+            schema: nonEmptyText(table.schema, `${label}.ownerCounting.schema`),
+            table: nonEmptyText(table.table, `${label}.ownerCounting.table`),
+        };
+    }
+    else if (kind === 'rows') {
+        const rows = record(owner, `${label}.ownerCounting`, ['kind', 'reason']);
+        ownerCounting = { kind: 'rows', reason: nonEmptyText(rows.reason, `${label}.ownerCounting.reason`) };
+    }
+    else {
+        fail(`${label}.ownerCounting.kind must be owner-table or rows.`);
+    }
+    return {
+        categoryMaxDistinct: positiveInteger(fields.categoryMaxDistinct, `${label}.categoryMaxDistinct`),
+        categoryMinOwners: positiveInteger(fields.categoryMinOwners, `${label}.categoryMinOwners`),
+        ownerCounting,
+    };
+}
+function parseColumn(value, label) {
+    const fields = record(value, label, [
+        'name', 'dataType', 'notNull', 'nullFraction', 'distinctEstimate', 'avgWidthBytes', 'kind',
+        'personalReason', 'categories', 'lengthQuantiles', 'numberQuantiles', 'booleanTrueFraction', 'shapes', 'jsonKeys', 'oddities',
+    ]);
+    const kind = oneOf(fields.kind, `${label}.kind`, COLUMN_KINDS);
+    const personal = kind === 'personal' || kind === 'free-text';
+    const personalReason = fields.personalReason === null ? null : nonEmptyText(fields.personalReason, `${label}.personalReason`);
+    if (personal && personalReason === null)
+        fail(`${label} is ${kind} but has no personalReason.`);
+    if (!personal && personalReason !== null)
+        fail(`${label} has a personalReason but its kind is ${kind}.`);
+    let categories = null;
+    if (fields.categories !== null) {
+        if (kind !== 'category')
+            fail(`${label} carries category values but its kind is ${kind}; values leave only for category columns.`);
+        categories = list(fields.categories, `${label}.categories`).map((item, index) => {
+            const entry = record(item, `${label}.categories[${index}]`, ['value', 'fraction', 'source']);
+            return {
+                value: text(entry.value, `${label}.categories[${index}].value`),
+                fraction: fraction(entry.fraction, `${label}.categories[${index}].fraction`),
+                source: oneOf(entry.source, `${label}.categories[${index}].source`, ['schema', 'data']),
+            };
+        });
+    }
+    else if (kind === 'category') {
+        fail(`${label} is a category column, so categories must be an array (empty when no value may leave).`);
+    }
+    let booleanTrueFraction = null;
+    if (fields.booleanTrueFraction !== null) {
+        if (kind !== 'boolean')
+            fail(`${label} carries booleanTrueFraction but its kind is ${kind}; it is only for boolean columns.`);
+        booleanTrueFraction = fraction(fields.booleanTrueFraction, `${label}.booleanTrueFraction`);
+    }
+    const jsonKeys = list(fields.jsonKeys, `${label}.jsonKeys`).map((item, index) => {
+        const entry = record(item, `${label}.jsonKeys[${index}]`, ['path', 'types', 'presentFraction']);
+        return {
+            path: nonEmptyText(entry.path, `${label}.jsonKeys[${index}].path`),
+            types: list(entry.types, `${label}.jsonKeys[${index}].types`)
+                .map((type, typeIndex) => nonEmptyText(type, `${label}.jsonKeys[${index}].types[${typeIndex}]`)),
+            presentFraction: fraction(entry.presentFraction, `${label}.jsonKeys[${index}].presentFraction`),
+        };
+    });
+    const keyPaths = new Set(jsonKeys.map(key => key.path));
+    return {
+        name: nonEmptyText(fields.name, `${label}.name`),
+        dataType: nonEmptyText(fields.dataType, `${label}.dataType`),
+        notNull: bool(fields.notNull, `${label}.notNull`),
+        nullFraction: fraction(fields.nullFraction, `${label}.nullFraction`),
+        distinctEstimate: nonNegative(fields.distinctEstimate, `${label}.distinctEstimate`),
+        avgWidthBytes: nonNegative(fields.avgWidthBytes, `${label}.avgWidthBytes`),
+        kind,
+        personalReason,
+        categories,
+        lengthQuantiles: fields.lengthQuantiles === null ? null : parseQuantiles(fields.lengthQuantiles, `${label}.lengthQuantiles`),
+        numberQuantiles: fields.numberQuantiles === null ? null : parseQuantiles(fields.numberQuantiles, `${label}.numberQuantiles`),
+        booleanTrueFraction,
+        shapes: list(fields.shapes, `${label}.shapes`).map((item, index) => {
+            const entry = record(item, `${label}.shapes[${index}]`, ['shape', 'fraction']);
+            return {
+                shape: text(entry.shape, `${label}.shapes[${index}].shape`),
+                fraction: fraction(entry.fraction, `${label}.shapes[${index}].fraction`),
+            };
+        }),
+        jsonKeys,
+        oddities: list(fields.oddities, `${label}.oddities`).map((item, index) => parseOddity(item, `${label}.oddities[${index}]`, keyPaths)),
+    };
+}
+/** One oddity: `jsonPath` must be one of the column's jsonKeys paths (so it
+ * passed the same owner and personal-key checks), set for a missing-json-key
+ * and never for oversize, invalid-encoding or shape-outlier; `types` is set
+ * exactly for a type-mismatch, with names from ODDITY_TYPE_NAMES. */
+function parseOddity(value, label, keyPaths) {
+    const entry = record(value, label, ['kind', 'rowCount', 'jsonPath', 'types', 'description']);
+    const kind = oneOf(entry.kind, `${label}.kind`, ODDITY_KINDS);
+    let jsonPath = null;
+    if (entry.jsonPath !== null) {
+        jsonPath = nonEmptyText(entry.jsonPath, `${label}.jsonPath`);
+        if (!keyPaths.has(jsonPath))
+            fail(`${label}.jsonPath is not one of the column's jsonKeys paths; an oddity names only a key path the profile already shows.`);
+        if (kind !== 'type-mismatch' && kind !== 'unexpected-null' && kind !== 'missing-json-key')
+            fail(`${label} is ${kind}, which is about the whole value, so its jsonPath must be null.`);
+    }
+    else if (kind === 'missing-json-key') {
+        fail(`${label} is a missing-json-key, so its jsonPath must name the key.`);
+    }
+    let types = null;
+    if (entry.types !== null) {
+        if (kind !== 'type-mismatch')
+            fail(`${label} carries types but its kind is ${kind}; types are only for type-mismatch.`);
+        const pair = record(entry.types, `${label}.types`, ['expected', 'found']);
+        const found = list(pair.found, `${label}.types.found`).map((name, index) => oneOf(name, `${label}.types.found[${index}]`, ODDITY_TYPE_NAMES));
+        if (found.length === 0)
+            fail(`${label}.types.found must name at least one type.`);
+        types = { expected: oneOf(pair.expected, `${label}.types.expected`, ODDITY_TYPE_NAMES), found };
+    }
+    else if (kind === 'type-mismatch') {
+        fail(`${label} is a type-mismatch, so types must give the expected and found types.`);
+    }
+    return {
+        kind,
+        rowCount: count(entry.rowCount, `${label}.rowCount`),
+        jsonPath,
+        types,
+        description: nonEmptyText(entry.description, `${label}.description`),
+    };
+}
+function tagged(value, label) {
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        fail(`${label} must be an object.`);
+    const fields = value;
+    return { fields, status: fields.status };
+}
+function parseSampling(value, label) {
+    const { status } = tagged(value, label);
+    if (status === 'full-scan') {
+        record(value, label, ['status']);
+        return { status: 'full-scan' };
+    }
+    if (status === 'sampled') {
+        const fields = record(value, label, ['status', 'percent', 'seed']);
+        const percent = finite(fields.percent, `${label}.percent`);
+        if (!(percent > 0 && percent < 100))
+            fail(`${label}.percent must be above 0 and below 100.`);
+        return { status: 'sampled', percent, seed: count(fields.seed, `${label}.seed`) };
+    }
+    return fail(`${label}.status must be full-scan or sampled.`);
+}
+function parseDeadRows(value, label) {
+    const { status } = tagged(value, label);
+    if (status === 'measured') {
+        const fields = record(value, label, ['status', 'fraction']);
+        return { status: 'measured', fraction: fraction(fields.fraction, `${label}.fraction`) };
+    }
+    if (status === 'unavailable') {
+        const fields = record(value, label, ['status', 'reason']);
+        return { status: 'unavailable', reason: nonEmptyText(fields.reason, `${label}.reason`) };
+    }
+    return fail(`${label}.status must be measured or unavailable.`);
+}
+function parseIndexDefinition(value, label) {
+    const definition = nonEmptyText(value, label);
+    let redacted;
+    try {
+        redacted = redactSqlLiterals(definition);
+    }
+    catch (error) {
+        if (error instanceof SqlLiteralError)
+            fail(`${label} cannot be checked for literal values: ${error.message}.`);
+        throw error;
+    }
+    if (redacted !== definition)
+        fail(`${label} still contains a literal value; every literal in an index definition must be replaced with ?.`);
+    return definition;
+}
+function parseTable(value, label) {
+    const fields = record(value, label, [
+        'schema', 'name', 'rowCount', 'rowCountSource', 'sampling', 'tableBytes', 'indexBytes', 'toastBytes', 'deadRows',
+        'columns', 'foreignKeys', 'indexes',
+    ]);
+    const schema = nonEmptyText(fields.schema, `${label}.schema`);
+    const name = nonEmptyText(fields.name, `${label}.name`);
+    const at = `${label} (${schema}.${name})`;
+    return {
+        schema,
+        name,
+        rowCount: count(fields.rowCount, `${at}.rowCount`),
+        rowCountSource: oneOf(fields.rowCountSource, `${at}.rowCountSource`, ['exact', 'estimate']),
+        sampling: parseSampling(fields.sampling, `${at}.sampling`),
+        tableBytes: count(fields.tableBytes, `${at}.tableBytes`),
+        indexBytes: count(fields.indexBytes, `${at}.indexBytes`),
+        toastBytes: count(fields.toastBytes, `${at}.toastBytes`),
+        deadRows: parseDeadRows(fields.deadRows, `${at}.deadRows`),
+        columns: list(fields.columns, `${at}.columns`).map((item, index) => parseColumn(item, `${at}.columns[${index}]`)),
+        foreignKeys: list(fields.foreignKeys, `${at}.foreignKeys`).map((item, index) => {
+            const key = `${at}.foreignKeys[${index}]`;
+            const entry = record(item, key, ['name', 'columns', 'references', 'childrenPerParent', 'orphanFraction']);
+            const references = record(entry.references, `${key}.references`, ['schema', 'table', 'columns']);
+            return {
+                name: nonEmptyText(entry.name, `${key}.name`),
+                columns: list(entry.columns, `${key}.columns`).map((column, columnIndex) => nonEmptyText(column, `${key}.columns[${columnIndex}]`)),
+                references: {
+                    schema: nonEmptyText(references.schema, `${key}.references.schema`),
+                    table: nonEmptyText(references.table, `${key}.references.table`),
+                    columns: list(references.columns, `${key}.references.columns`)
+                        .map((column, columnIndex) => nonEmptyText(column, `${key}.references.columns[${columnIndex}]`)),
+                },
+                childrenPerParent: parseQuantiles(entry.childrenPerParent, `${key}.childrenPerParent`),
+                orphanFraction: fraction(entry.orphanFraction, `${key}.orphanFraction`),
+            };
+        }),
+        indexes: list(fields.indexes, `${at}.indexes`).map((item, index) => {
+            const entry = record(item, `${at}.indexes[${index}]`, ['name', 'definition']);
+            return {
+                name: nonEmptyText(entry.name, `${at}.indexes[${index}].name`),
+                definition: parseIndexDefinition(entry.definition, `${at}.indexes[${index}].definition`),
+            };
+        }),
+    };
+}
+/** Strictly parse a `DatabaseProfileV1`. */
+export function parseDatabaseProfile(value, label = 'profile') {
+    const fields = record(value, label, [
+        'version', 'profilerVersion', 'createdAt', 'engine', 'engineVersion', 'settings', 'rules', 'tables',
+    ]);
+    if (fields.version === 'dv-database-profile-v1')
+        fail(`${label}: ${OLDER_PROFILE_MESSAGE}.`);
+    if (fields.version !== DATABASE_PROFILE_VERSION) {
+        fail(`${label}.version must be ${DATABASE_PROFILE_VERSION}; this file is not a Haystack database profile this CLI understands.`);
+    }
+    return {
+        version: DATABASE_PROFILE_VERSION,
+        profilerVersion: nonEmptyText(fields.profilerVersion, `${label}.profilerVersion`),
+        createdAt: isoTime(fields.createdAt, `${label}.createdAt`),
+        engine: oneOf(fields.engine, `${label}.engine`, ['postgres']),
+        engineVersion: nonEmptyText(fields.engineVersion, `${label}.engineVersion`),
+        settings: list(fields.settings, `${label}.settings`).map((item, index) => {
+            const entry = record(item, `${label}.settings[${index}]`, ['name', 'value']);
+            return {
+                name: nonEmptyText(entry.name, `${label}.settings[${index}].name`),
+                value: text(entry.value, `${label}.settings[${index}].value`),
+            };
+        }),
+        rules: parseRules(fields.rules, `${label}.rules`),
+        tables: list(fields.tables, `${label}.tables`).map((item, index) => parseTable(item, `${label}.tables[${index}]`)),
+    };
+}
+/** Strictly parse the server's `DatabaseProfileUploadV1` response. */
+export function parseDatabaseProfileUpload(value, label = 'upload response') {
+    const fields = record(value, label, ['version', 'tenantId', 'repositoryId', 'dependencyId', 'profile', 'uploadedAt']);
+    if (fields.version !== DATABASE_PROFILE_UPLOAD_VERSION) {
+        fail(`${label}.version must be ${DATABASE_PROFILE_UPLOAD_VERSION}.`);
+    }
+    const artifact = record(fields.profile, `${label}.profile`, ['artifactId', 'uri', 'sha256']);
+    const sha256 = nonEmptyText(artifact.sha256, `${label}.profile.sha256`);
+    if (!/^[0-9a-f]{64}$/u.test(sha256))
+        fail(`${label}.profile.sha256 must be 64 lowercase hex characters.`);
+    return {
+        version: DATABASE_PROFILE_UPLOAD_VERSION,
+        tenantId: nonEmptyText(fields.tenantId, `${label}.tenantId`),
+        repositoryId: nonEmptyText(fields.repositoryId, `${label}.repositoryId`),
+        dependencyId: fields.dependencyId === null ? null : nonEmptyText(fields.dependencyId, `${label}.dependencyId`),
+        profile: {
+            artifactId: nonEmptyText(artifact.artifactId, `${label}.profile.artifactId`),
+            uri: nonEmptyText(artifact.uri, `${label}.profile.uri`),
+            sha256,
+        },
+        uploadedAt: isoTime(fields.uploadedAt, `${label}.uploadedAt`),
+    };
+}