crewhaus 0.1.8 → 0.2.1

package/dist/regression-pin.js

@@ -0,0 +1,281 @@
+/**
+ * Item 9 — pin every recovered failure into a permanent regression suite.
+ *
+ * When an optimize run is accepted, the samples that flipped fail→pass
+ * between the baseline eval run and the winning candidate's eval run are
+ * exactly the behaviors the accepted patch fixed. If the training dataset
+ * later churns, those regressions can silently return — so we pin them
+ * into a per-spec `<specName>-regressions` dataset in the Section 29
+ * registry (single `train` split; each pin unions the previous latest
+ * version's samples with the new recoveries, deduped by sample id), and
+ * `crewhaus eval` unions that suite into the loaded dataset by default.
+ *
+ * Kept in a side-effect-free module (the CLI entry file runs an argv
+ * switch on import) mirroring `eval-history.ts` / `datasets.ts`: all
+ * filesystem access goes through an injected `DatasetRegistry` or through
+ * eval-report's `loadRun` on caller-supplied run dirs. The failure-arbiter
+ * wiring (item 7, `triage.ts`) is the other consumer of
+ * {@link pinRecoveredSamples}: post-eval triage promotes the samples whose
+ * `ArbiterAction.promoteRegression` flag is set (bug-class failures,
+ * eval-optimizer-orchestrator/src/failure-arbiter.ts) with
+ * `source: "failure-arbiter"` provenance.
+ */
+import { createHash } from "node:crypto";
+import { latestVersion } from "@crewhaus/dataset-registry";
+import { diffReports, loadRun } from "@crewhaus/eval-report";
+import { nextVersion, overallDatasetHash } from "./datasets";
+/** The per-spec regression suite's registry name. */
+export function regressionSuiteName(specName) {
+    return `${specName}-regressions`;
+}
+/**
+ * dataset-registry's name grammar (spec `safeName` additionally allows
+ * spaces/colons, which the registry rejects). A spec whose name can't form
+ * a valid registry dataset name simply has no regression suite — both the
+ * pin and the union become no-ops instead of surfacing a registry error
+ * (with a one-line warning per invocation, so the no-op is never silent).
+ */
+const REGISTRY_NAME_REGEX = /^[A-Za-z0-9_-][A-Za-z0-9_.-]*$/;
+export function isRegistrySafeName(name) {
+    return REGISTRY_NAME_REGEX.test(name);
+}
+/** Default warning sink for the registry-unsafe-name no-ops. */
+const warnToStderr = (line) => {
+    process.stderr.write(`${line}\n`);
+};
+/** The one-line "this degraded to a no-op" warning for a spec whose name
+ *  can't map to the registry grammar (see {@link isRegistrySafeName}). */
+function unsafeNameWarning(what, specName, suiteName) {
+    return (`[eval] ${what} skipped: spec name "${specName}" can't form a registry dataset name ` +
+        `("${suiteName}") — rename the spec to match ${REGISTRY_NAME_REGEX} to enable the regression suite`);
+}
+/**
+ * Append recovered samples to the per-spec regression suite as a new
+ * auto-bumped version: union of the previous latest version's samples and
+ * the new ones, deduped by sample id (existing pins win, so re-pinning the
+ * same recovery is idempotent and keeps its original provenance). All
+ * samples live in the `train` split — the suite is a single-split dataset.
+ * Provenance (optimize runId, pin date, source dataset) is recorded in each
+ * newly pinned sample's `metadata.regression_pin` (SampleSchema carries an
+ * open `metadata` record). No new samples → no-op: no version is written.
+ */
+export async function pinRecoveredSamples(opts) {
+    const suiteName = regressionSuiteName(opts.specName);
+    if (opts.samples.length === 0)
+        return { suiteName, pinned: 0 };
+    if (!isRegistrySafeName(suiteName)) {
+        // There WERE samples to pin — surface the degradation instead of
+        // silently dropping them (one warning per invocation).
+        (opts.warn ?? warnToStderr)(unsafeNameWarning("regression pin", opts.specName, suiteName));
+        return { suiteName, pinned: 0 };
+    }
+    const prevVersion = await latestVersion(opts.registry, suiteName);
+    const prevTrain = prevVersion !== undefined
+        ? (await opts.registry.getRecord(suiteName, prevVersion)).splits.train
+        : [];
+    const byId = new Map(prevTrain.map((s) => [s.id, s]));
+    const pinnedAt = (opts.now?.() ?? new Date()).toISOString();
+    let pinned = 0;
+    for (const s of opts.samples) {
+        if (byId.has(s.id))
+            continue;
+        byId.set(s.id, {
+            ...s,
+            metadata: {
+                ...(s.metadata ?? {}),
+                regression_pin: {
+                    optimizeRunId: opts.optimizeRunId,
+                    pinnedAt,
+                    sourceDataset: opts.sourceDataset,
+                    ...(opts.source !== undefined ? { source: opts.source } : {}),
+                },
+            },
+        });
+        pinned += 1;
+    }
+    if (pinned === 0)
+        return { suiteName, pinned: 0 };
+    const version = nextVersion(await opts.registry.list(suiteName));
+    await opts.registry.put({
+        name: suiteName,
+        version,
+        splits: { train: [...byId.values()], dev: [] },
+    });
+    return { suiteName, pinned, version };
+}
+/**
+ * The optimize success path: diff the baseline eval run against the winning
+ * candidate's, extract `recoveries` (fail→pass sampleIds), look those up in
+ * the dataset that was used, and pin them via {@link pinRecoveredSamples}.
+ * Returns undefined when pinning is disabled or either run dir is missing
+ * (e.g. a fitness fn that doesn't report `runDir`).
+ */
+export async function pinRecoveriesAfterOptimize(opts) {
+    if (!opts.pin)
+        return undefined;
+    if (opts.baselineRunDir === undefined || opts.candidateRunDir === undefined)
+        return undefined;
+    const prev = await loadRun(opts.baselineRunDir);
+    const next = await loadRun(opts.candidateRunDir);
+    const { diff } = diffReports(prev, next);
+    const recovered = [];
+    for (const r of diff.recoveries) {
+        const s = opts.samplesById.get(r.sampleId);
+        if (s !== undefined)
+            recovered.push(s);
+    }
+    return pinRecoveredSamples({
+        registry: opts.registry,
+        specName: opts.specName,
+        samples: recovered,
+        sourceDataset: opts.sourceDataset,
+        optimizeRunId: opts.optimizeRunId,
+        ...(opts.now !== undefined ? { now: opts.now } : {}),
+    });
+}
+/**
+ * Union the per-spec regression suite into an eval's dataset (dedupe by
+ * sample id; the primary dataset wins on collision). Returns undefined when
+ * disabled, when no suite exists, or when the primary IS the suite (a
+ * `registry:` ref to it, or a file whose basename matches it — an exported
+ * copy must not union the suite into itself).
+ *
+ * Item-3 interaction, by design: a union that ADDS samples changes the
+ * run's sample keyset, so the returned `datasetName` is suffixed
+ * `+regressions@vX` and the returned `datasetHash` folds the suite's
+ * content hash into the primary's. Run-index entries and (spec, dataset)
+ * baselines then key on the honest union lineage — the first ADDING union
+ * starts a new baseline lineage instead of tripping diffReports'
+ * keyset-mismatch guard against the pre-union baseline. A union that adds
+ * NOTHING (every suite sample already in the primary) keeps the primary
+ * identity untouched: the keyset didn't change, so rewriting the key would
+ * orphan the existing baseline and silently disarm the gate.
+ */
+export async function applyRegressionUnion(opts) {
+    if (!opts.includeRegressions)
+        return undefined;
+    const suiteName = regressionSuiteName(opts.specName);
+    if (!isRegistrySafeName(suiteName)) {
+        (opts.warn ?? warnToStderr)(unsafeNameWarning("regression suite union", opts.specName, suiteName));
+        return undefined;
+    }
+    if (opts.primaryRegistryName === suiteName)
+        return undefined;
+    // File-dataset self-union guard: a file literally named after the suite
+    // (`<specName>-regressions.jsonl` — loadDataset names it by basename) is
+    // treated as an export of the suite, same as the registry ref above.
+    if (opts.datasetName === suiteName || opts.datasetName.replace(/\.[^.]+$/, "") === suiteName) {
+        return undefined;
+    }
+    const suiteVersion = await latestVersion(opts.registry, suiteName);
+    if (suiteVersion === undefined)
+        return undefined;
+    const record = await opts.registry.getRecord(suiteName, suiteVersion);
+    // Validate the record shape BEFORE materializing the primary stream — a
+    // corrupt suite record must fail here, while the caller's fallback can
+    // still hand the untouched primary stream to the runner.
+    if (!Array.isArray(record?.splits?.train)) {
+        throw new Error(`regression suite ${suiteName}@${suiteVersion} is corrupt (missing splits.train) — re-pin it or delete the record`);
+    }
+    const primary = await opts.loadPrimarySamples();
+    const seen = new Set(primary.map((s) => s.id));
+    const samples = [...primary];
+    let added = 0;
+    for (const s of record.splits.train) {
+        if (seen.has(s.id))
+            continue;
+        seen.add(s.id);
+        samples.push(s);
+        added += 1;
+    }
+    if (added === 0) {
+        // Keyset unchanged — keep the primary identity so the lineage stays
+        // comparable (see the doc comment above).
+        return {
+            samples,
+            datasetName: opts.datasetName,
+            datasetHash: opts.datasetHash,
+            added,
+            suiteName,
+            suiteVersion,
+        };
+    }
+    return {
+        samples,
+        datasetName: `${opts.datasetName}+regressions@${suiteVersion}`,
+        datasetHash: foldDatasetHash(opts.datasetHash, overallDatasetHash(record, ["train"])),
+        added,
+        suiteName,
+        suiteVersion,
+    };
+}
+/**
+ * `applyRegressionUnion` with the failure semantics `crewhaus eval` needs:
+ * best-effort (a broken suite record warns and falls back to the primary
+ * dataset) AND stream-loss-proof. The primary samples are captured as they
+ * are materialized for the union attempt, so a throw AFTER the one-shot
+ * primary stream was consumed (e.g. a suite record that passes the shape
+ * check but breaks the hash fold) falls back to the captured array instead
+ * of handing the runner an exhausted iterable — which would silently run a
+ * 0-sample eval.
+ */
+export async function applyRegressionUnionGuarded(opts) {
+    const warn = opts.warn ?? warnToStderr;
+    let materialized;
+    try {
+        const union = await applyRegressionUnion({
+            registry: opts.registry,
+            specName: opts.specName,
+            includeRegressions: opts.includeRegressions,
+            ...(opts.primaryRegistryName !== undefined
+                ? { primaryRegistryName: opts.primaryRegistryName }
+                : {}),
+            loadPrimarySamples: async () => {
+                materialized = [];
+                for await (const s of opts.primary.samples)
+                    materialized.push(s);
+                return materialized;
+            },
+            datasetName: opts.primary.name,
+            datasetHash: opts.datasetHash,
+            warn,
+        });
+        if (union !== undefined) {
+            return {
+                datasetName: union.datasetName,
+                datasetHash: union.datasetHash,
+                samples: reiterable(union.samples),
+                union,
+            };
+        }
+    }
+    catch (err) {
+        warn(`[eval] regression suite union skipped: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    return {
+        datasetName: opts.primary.name,
+        datasetHash: opts.datasetHash,
+        samples: materialized !== undefined ? reiterable(materialized) : opts.primary.samples,
+    };
+}
+/** Re-iterable view over an already-materialized sample array. */
+function reiterable(samples) {
+    return {
+        async *[Symbol.asyncIterator]() {
+            for (const s of samples)
+                yield s;
+        },
+    };
+}
+/**
+ * One stable sha256 folding the regression suite's content hash into the
+ * primary dataset's, with domain separators (mirrors `overallDatasetHash`'s
+ * style). Keeps RunIndexEntry.datasetHash honest for unioned runs: it
+ * changes when EITHER the primary bytes or the suite content change.
+ */
+export function foldDatasetHash(primaryHash, suiteHash) {
+    return createHash("sha256")
+        .update(`primary:${primaryHash}\n`)
+        .update(`regressions:${suiteHash}\n`)
+        .digest("hex");
+}

package/dist/retention.d.ts

@@ -0,0 +1,193 @@
+import { type PurgeResult, type RecordStore, type RetentionConfig, type RetentionRecord } from "@crewhaus/data-retention-engine";
+export { DEFAULT_SESSION_MAX_AGE_DAYS, RETENTION_CONFIG_RELPATH, type RetentionAuditWindowConfig, type RetentionConfig, RetentionConfigError, loadRetentionConfig, } from "@crewhaus/data-retention-engine";
+/** Single-tenant placeholder: the on-disk stores carry no tenant ids. */
+export declare const LOCAL_TENANT = "local";
+/**
+ * Why NO audit data — not individual records, not whole day files — is ever
+ * deleted by retention. Kept as an exported constant so the report, the
+ * adapter refusal, and the tests all cite the same rationale.
+ */
+export declare const AUDIT_CHAIN_EXCLUSION_REASON: string;
+/** Thrown by `parseRetentionDate` on an unparseable `--since`/`--before`. */
+export declare class InvalidRetentionDateError extends Error {
+    readonly name = "InvalidRetentionDateError";
+    constructor(flag: string, value: string);
+}
+/** Parse a `--since`/`--before` flag value to epoch ms (UTC for bare dates). */
+export declare function parseRetentionDate(flag: string, value: string): number;
+export type SkippedEntry = {
+    readonly path: string;
+    readonly reason: string;
+};
+export type DeletionEntry = {
+    readonly id: string;
+    readonly paths: ReadonlyArray<string>;
+};
+export type RefusalEntry = {
+    readonly id: string;
+    readonly reason: string;
+};
+type SessionInventoryEntry = {
+    readonly record: RetentionRecord;
+    readonly sessionId: string;
+    readonly jsonPath: string;
+    /** Present when the sibling `<id>.jsonl` event log exists. */
+    readonly eventLogPath?: string;
+};
+type AuditInventoryEntry = {
+    readonly record: RetentionRecord;
+    readonly day: string;
+    readonly path: string;
+};
+/**
+ * RecordStore adapter over `<rootDir>/.crewhaus/sessions` +
+ * `<rootDir>/.crewhaus/audit`. The inventory is a snapshot taken by
+ * `openHarnessRecordStore` (a sweep operates on the state it observed at
+ * start). Deletions are tracked for reporting; in `dryRun` mode `delete()`
+ * makes the same decision (and records it) without touching the disk, so the
+ * engine's counters mirror a real run exactly.
+ */
+export declare class HarnessRecordStore implements RecordStore {
+    readonly sessions: ReadonlyArray<SessionInventoryEntry>;
+    readonly auditDays: ReadonlyArray<AuditInventoryEntry>;
+    readonly skipped: ReadonlyArray<SkippedEntry>;
+    readonly deletions: DeletionEntry[];
+    readonly refusals: RefusalEntry[];
+    private readonly byId;
+    private readonly pins;
+    private readonly dryRun;
+    private readonly sessionsDir;
+    constructor(opts: {
+        readonly sessionsDir: string;
+        readonly sessions: ReadonlyArray<SessionInventoryEntry>;
+        readonly auditDays: ReadonlyArray<AuditInventoryEntry>;
+        readonly skipped: ReadonlyArray<SkippedEntry>;
+        readonly pins: ReadonlyArray<string>;
+        readonly dryRun: boolean;
+    });
+    listAll(): AsyncIterable<RetentionRecord>;
+    listByTenant(tenantId: string): AsyncIterable<RetentionRecord>;
+    delete(id: string): Promise<boolean>;
+}
+export type OpenHarnessRecordStoreOptions = {
+    readonly rootDir: string;
+    readonly pins?: ReadonlyArray<string>;
+    readonly dryRun?: boolean;
+};
+/**
+ * Snapshot the two stores under `rootDir` into a `HarnessRecordStore`.
+ * Anything unrecognised is SKIPPED with a reason, never deleted:
+ * `.tmp` leftovers, non-`sess_` filenames, orphaned event logs (a
+ * `<id>.jsonl` without its `<id>.json`), and non-day files in the audit dir.
+ */
+export declare function openHarnessRecordStore(opts: OpenHarnessRecordStoreOptions): Promise<HarnessRecordStore>;
+export type EvidenceRef = {
+    readonly seq: number;
+    readonly hash: string;
+};
+export type RetentionEnforcementReport = {
+    readonly action: "sweep" | "purge";
+    readonly rootDir: string;
+    readonly dryRun: boolean;
+    readonly config: RetentionConfig;
+    readonly before?: number;
+    /** Sessions deleted (or, dry-run, that WOULD be) with their on-disk paths. */
+    readonly deleted: ReadonlyArray<DeletionEntry>;
+    /** Sessions kept because they are pinned (only reached once expired). */
+    readonly keptPinned: ReadonlyArray<RefusalEntry>;
+    /** Audit day files — always kept; see AUDIT_CHAIN_EXCLUSION_REASON. */
+    readonly keptAuditChain: ReadonlyArray<string>;
+    /** Session ids still inside the retention window (age < maxAgeDays). */
+    readonly keptWithinRetention: ReadonlyArray<string>;
+    /** Record ids deferred because an audit window is active. */
+    readonly keptAuditWindow: ReadonlyArray<string>;
+    /** Session ids outside the `--before` cutoff (purge only). */
+    readonly keptOutsideCutoff: ReadonlyArray<string>;
+    readonly activeAuditWindows: PurgeResult["auditWindowDeferred"];
+    readonly skipped: ReadonlyArray<SkippedEntry>;
+    /** The appended `retention_enforcement` record (real runs only). */
+    readonly evidence?: EvidenceRef;
+};
+export type RunRetentionSweepOptions = {
+    readonly rootDir: string;
+    readonly dryRun?: boolean;
+    readonly now?: () => number;
+};
+/**
+ * `crewhaus retention sweep` — the scheduled GDPR/TTL enforcement pass.
+ * Deletes expired sessions (mtime older than `sessions.maxAgeDays`), honors
+ * pins + audit windows, never touches audit data, and appends a
+ * `retention_enforcement` evidence record on real runs. Exported as a plain
+ * library function so a daemon janitor can call it without the CLI.
+ */
+export declare function runRetentionSweep(opts: RunRetentionSweepOptions): Promise<RetentionEnforcementReport>;
+export type RunRetentionPurgeOptions = {
+    readonly rootDir: string;
+    /** Only records with createdAt strictly BEFORE this epoch-ms cutoff are
+     *  candidates. Omitted = every expired record (same coverage as sweep). */
+    readonly before?: number;
+    /** Report what WOULD be purged; delete nothing, append no evidence.
+     *  (Ops-review F1: `--dry-run` used to be accepted-and-ignored here — a
+     *  dry run that really deletes is the worst possible failure mode.) */
+    readonly dryRun?: boolean;
+    readonly now?: () => number;
+};
+/**
+ * `crewhaus retention purge` — the explicit right-to-delete verb. Identical
+ * rules to the sweep (pins, audit windows, retention TTLs, audit exclusion)
+ * restricted to records older than `--before` when given. Not a bypass:
+ * a record still inside its retention window is deferred, not deleted.
+ */
+export declare function runRetentionPurge(opts: RunRetentionPurgeOptions): Promise<RetentionEnforcementReport>;
+export type RetentionExportReport = {
+    readonly rootDir: string;
+    readonly outDir: string;
+    readonly dryRun: boolean;
+    readonly since?: number;
+    /** Session ids whose files were copied (dry run: WOULD be copied). */
+    readonly sessions: ReadonlyArray<string>;
+    /** Audit day labels whose files were copied (dry run: WOULD be copied). */
+    readonly auditDays: ReadonlyArray<string>;
+    /**
+     * True when the export covered EVERY audit day file and the on-host
+     * `_chain-tail.json` anchor was copied too — the exported audit directory
+     * is then independently verifiable with `crewhaus audit verify --dir
+     * <outDir>/audit`. A `--since`-filtered (partial) export omits the anchor:
+     * its chain starts mid-stream and would fail verification by construction.
+     * (Dry run: whether the anchor WOULD be copied.)
+     */
+    readonly chainTailCopied: boolean;
+    readonly skipped: ReadonlyArray<SkippedEntry>;
+    /** The appended `retention_enforcement` record. Absent on a dry run, and
+     *  on a real no-op export against a harness with no audit store (F8b). */
+    readonly evidence?: EvidenceRef;
+};
+export type RunRetentionExportOptions = {
+    readonly rootDir: string;
+    readonly outDir: string;
+    /** Only records at/after this epoch-ms timestamp are exported (sessions by
+     *  mtime; audit at day granularity — a day is included when any instant of
+     *  it is >= since, an inclusive over-approximation). */
+    readonly since?: number;
+    /** Report what WOULD be exported; write nothing (no outDir, no manifest),
+     *  append no audit evidence. */
+    readonly dryRun?: boolean;
+    readonly now?: () => number;
+};
+/**
+ * `crewhaus retention export` — the right-to-export verb. Copies matching
+ * records OUT of the harness stores (originals untouched) as raw files, so
+ * an exported audit set keeps its verbatim hash-chain lines:
+ *
+ *   <outDir>/sessions/<id>.json (+ <id>.jsonl event log when present)
+ *   <outDir>/audit/<day>.jsonl  (+ _chain-tail.json on a complete export)
+ *   <outDir>/manifest.json
+ *
+ * Non-destructive, so pins and audit windows do not gate it; it IS
+ * audit-logged (a `retention_enforcement` record with action "export").
+ */
+export declare function runRetentionExport(opts: RunRetentionExportOptions): Promise<RetentionExportReport>;
+/** Render a sweep/purge report as the CLI's indented summary lines. */
+export declare function formatEnforcementReport(report: RetentionEnforcementReport): ReadonlyArray<string>;
+/** Render an export report as the CLI's indented summary lines. */
+export declare function formatExportReport(report: RetentionExportReport): ReadonlyArray<string>;