crewhaus 0.1.8 → 0.2.1

package/dist/audit-verify.js

@@ -0,0 +1,97 @@
+const VALID_ANCHOR_SCHEMES = ["file"];
+/** Thrown by `resolveAnchorFlag` on a malformed `--anchor` value. The CLI
+ *  entry file catches it and routes the message through `die()`; tests
+ *  assert on `.message` without the process exiting. */
+export class InvalidAnchorFlagError extends Error {
+    value;
+    name = "InvalidAnchorFlagError";
+    constructor(value) {
+        super(`invalid --anchor "${value}" — expected <scheme>:<path> with scheme one of: ${VALID_ANCHOR_SCHEMES.join(", ")} (e.g. file:/var/crewhaus/anchors)`);
+        this.value = value;
+    }
+}
+/**
+ * Parse the raw `--anchor` flag value (or undefined when absent). `file:<path>`
+ * selects the file-backed AnchorStore rooted at `<path>`; anything else —
+ * missing scheme, empty path, unknown scheme (S3 et al. are out of scope) —
+ * throws {@link InvalidAnchorFlagError}.
+ */
+export function resolveAnchorFlag(flagValue) {
+    if (flagValue === undefined)
+        return undefined;
+    const sep = flagValue.indexOf(":");
+    if (sep <= 0)
+        throw new InvalidAnchorFlagError(flagValue);
+    const scheme = flagValue.slice(0, sep);
+    const path = flagValue.slice(sep + 1);
+    if (scheme !== "file" || path === "")
+        throw new InvalidAnchorFlagError(flagValue);
+    return { scheme, path };
+}
+/**
+ * Render a `VerifyResult` as per-check summary lines + the process exit code.
+ * `anchorRequested` is whether the user passed `--anchor`: when true, an
+ * `ok` result whose external anchor was NOT actually consulted (store empty,
+ * unreadable, or lagging into nonexistence) is a FAILURE — the check the
+ * user scheduled did not run, so exiting 0 would fabricate assurance.
+ */
+export function summarizeVerifyResult(result, opts) {
+    if (!result.ok) {
+        return {
+            exitCode: 1,
+            lines: [
+                `✗ tamper finding at ${result.file}:${result.line} — ${result.reason}`,
+                `  ${result.recordsChecked} record(s) verified cleanly before the finding`,
+            ],
+        };
+    }
+    const lines = [
+        `✓ hash chain intact — ${result.recordsChecked} record(s) checked (per-record hash, prevHash links, gapless seq from 0)`,
+    ];
+    if (result.anchorChecked) {
+        lines.push("✓ on-host chain-tail anchor matches the surviving tail");
+    }
+    else {
+        lines.push("~ on-host chain-tail anchor absent — tail truncation cannot be ruled out (limitation, not tamper)");
+    }
+    let exitCode = 0;
+    if (result.externalAnchorChecked) {
+        lines.push("✓ external anchor store agrees with the chain tip");
+    }
+    else if (opts.anchorRequested) {
+        exitCode = 1;
+        lines.push("✗ external anchor requested (--anchor) but could not be cross-checked — the store holds no anchor for this log (or is unreadable), so the strongest tamper witness did not run");
+    }
+    else {
+        lines.push("~ external anchor not consulted (pass --anchor file:<path> to cross-check one)");
+    }
+    return { lines, exitCode };
+}
+/**
+ * Map a `VerifyResult` onto the doctor's ✓/~/✗ check shape so `crewhaus
+ * doctor` can surface audit-log tampering when a `.crewhaus/audit` store
+ * exists in the cwd. An intact chain without the on-host anchor is a warn
+ * ("~"), never a failure — mirroring the library's limitation-not-tamper
+ * stance — and a broken chain fails doctor with the first finding inline.
+ */
+export function buildAuditIntegrityCheck(result) {
+    if (!result.ok) {
+        return {
+            label: "Audit log integrity (.crewhaus/audit)",
+            pass: false,
+            reason: `${result.reason} (${result.file}:${result.line}) — run \`crewhaus audit verify\` for details`,
+        };
+    }
+    if (!result.anchorChecked) {
+        return {
+            label: `Audit log integrity (.crewhaus/audit, ${result.recordsChecked} records)`,
+            pass: true,
+            warn: true,
+            reason: "chain intact but the chain-tail anchor is absent — tail truncation cannot be ruled out",
+        };
+    }
+    return {
+        label: `Audit log integrity (.crewhaus/audit, ${result.recordsChecked} records)`,
+        pass: true,
+    };
+}

package/dist/autodistill.d.ts

@@ -0,0 +1,113 @@
+import type { DatasetRegistry } from "@crewhaus/dataset-registry";
+import type { IrFeedback } from "@crewhaus/ir";
+import { type DerivedTurn, type FeedbackRecord, type SessionTurn } from "./feedback";
+/** Default "≥ N unprocessed ratings" trigger (the spec's `autoDistill` is a
+ *  plain boolean and carries no threshold of its own). */
+export declare const DEFAULT_AUTODISTILL_THRESHOLD = 5;
+/** Env override for the trigger threshold. */
+export declare const AUTODISTILL_THRESHOLD_ENV = "CREWHAUS_AUTODISTILL_THRESHOLD";
+/** Opt-out env for the REPL exit rating prompt. */
+export declare const NO_EXIT_RATING_ENV = "CREWHAUS_NO_EXIT_RATING";
+/** The watermark file, relative to the harness cwd. */
+export declare const DISTILL_STATE_RELPATH: string;
+/** The registry dataset autoDistill maintains for a spec. */
+export declare function ratingsDatasetName(specName: string): string;
+export type DistillState = {
+    readonly schemaVersion: 1;
+    /** ISO ts of the newest feedback record folded into the last auto-distill;
+     *  records with a STRICTLY greater ts count as unprocessed. */
+    readonly lastProcessedTs: string;
+    /** How many records the last auto-distill saw in total (informational). */
+    readonly processedCount: number;
+    /** The last registry version this consumer produced (informational). */
+    readonly lastRegistered?: {
+        readonly name: string;
+        readonly version: string;
+        readonly at: string;
+    };
+};
+/** Narrow parsed JSON to a DistillState; undefined on any malformation (a
+ *  corrupt watermark degrades to "everything is unprocessed", never throws). */
+export declare function parseDistillState(text: string): DistillState | undefined;
+export declare function readDistillState(path: string): DistillState | undefined;
+export declare function writeDistillState(path: string, state: DistillState): void;
+/** Records newer than the watermark (strict ISO-string comparison — the
+ *  capture surfaces all stamp `new Date().toISOString()`, which compares
+ *  lexicographically). No watermark → everything is unprocessed. */
+export declare function countUnprocessed(records: ReadonlyArray<FeedbackRecord>, lastProcessedTs: string | undefined): number;
+export declare function newestTs(records: ReadonlyArray<FeedbackRecord>): string | undefined;
+export declare function resolveAutoDistillThreshold(env: Readonly<Record<string, string | undefined>>): number;
+export type AutoDistillDecision = {
+    readonly run: boolean;
+    readonly reason: string;
+};
+/** Whether the teardown should distill: the spec opted in
+ *  (`feedback.autoDistill: true`, block not disabled) AND enough
+ *  unprocessed ratings accumulated. */
+export declare function shouldAutoDistill(opts: {
+    readonly feedback: IrFeedback | undefined;
+    readonly unprocessed: number;
+    readonly threshold: number;
+}): AutoDistillDecision;
+export type MaybeAutoDistillOptions = {
+    readonly specName: string;
+    readonly feedback: IrFeedback | undefined;
+    /** All sessions' turns (tagged with the sessionId join key). */
+    readonly turns: ReadonlyArray<SessionTurn>;
+    /** The accumulated store: in-transcript `user_feedback` records plus the
+     *  web-UI `.crewhaus/feedback/*.jsonl` records. */
+    readonly records: ReadonlyArray<FeedbackRecord>;
+    readonly registry: DatasetRegistry;
+    /** Watermark file path (see {@link DISTILL_STATE_RELPATH}). */
+    readonly stateFilePath: string;
+    /** Env for the threshold/opt-outs; defaults to process.env. */
+    readonly env?: Readonly<Record<string, string | undefined>>;
+    /** Positive-sample cutoff forwarded to distill(); default 0.7 (matches
+     *  `crewhaus distill --min-score`). */
+    readonly minScore?: number;
+    readonly now?: () => Date;
+    /** Line sink; defaults to stdout. */
+    readonly write?: (line: string) => void;
+};
+export type MaybeAutoDistillResult = {
+    readonly ran: boolean;
+    readonly reason: string;
+    readonly registered?: {
+        readonly name: string;
+        readonly version: string;
+    };
+};
+/**
+ * The teardown consumer. One printed line per outcome that did anything;
+ * silent when the trigger simply hasn't accumulated yet. Best-effort by
+ * contract — the CALLER wraps it so a failure can never turn a successful
+ * session into a non-zero exit.
+ */
+export declare function maybeAutoDistill(opts: MaybeAutoDistillOptions): Promise<MaybeAutoDistillResult>;
+export declare const EXIT_RATING_PROMPT = "rate this session? [g]ood / [b]ad / [enter] skip: ";
+export declare const EXIT_RATING_TIMEOUT_MS = 10000;
+/** Turns with a non-empty assistant answer — the "session actually said
+ *  something worth rating" floor for the exit prompt. */
+export declare function countAssistantTurns(turns: ReadonlyArray<DerivedTurn>): number;
+export type ExitRatingDecision = {
+    readonly prompt: boolean;
+    readonly reason: string;
+};
+/**
+ * The pure gate for the exit prompt. Prompts only when ALL hold: the spec
+ * has a feedback block (presence opts in), it isn't disabled
+ * (`enabled: false`) or prompt-opted-out (`exitPrompt: false`), the
+ * CREWHAUS_NO_EXIT_RATING env is not set, stdin is a real TTY (NEVER in
+ * piped/CI mode), and the session produced at least one assistant answer.
+ */
+export declare function shouldPromptExitRating(opts: {
+    readonly stdinIsTTY: boolean;
+    readonly env: Readonly<Record<string, string | undefined>>;
+    readonly feedback: IrFeedback | undefined;
+    readonly assistantTurns: number;
+}): ExitRatingDecision;
+export type ExitRatingChoice = "up" | "down" | "skip";
+/** Map the raw keystroke to a rating: g/G → good (thumbs up), b/B → bad
+ *  (thumbs down), anything else — enter, timeout (undefined), Ctrl-C/D —
+ *  skips. One keystroke, no confirmation, zero-cost to ignore. */
+export declare function parseExitRatingKey(key: string | undefined): ExitRatingChoice;

package/dist/autodistill.js

@@ -0,0 +1,256 @@
+/**
+ * Item 1 — the `feedback.autoDistill` consumer + the REPL exit-rating
+ * gating logic. Gives the parsed-but-inert spec flag its first reader:
+ *
+ *   - **autoDistill**: at CLI `run` teardown (NOT a spawned hook — hooks
+ *     run credential-stripped), when the accumulated feedback store holds
+ *     ≥ N unprocessed `user_feedback` records, the existing `distill()`
+ *     runs over ALL sessions and the result is registered as a new
+ *     auto-bumped version of the `<specName>-ratings` registry dataset —
+ *     immediately consumable as `--dataset registry:<specName>-ratings`
+ *     (the item-12 shorthand `eval`/`optimize` already speak). The spec's
+ *     `autoDistill` is a plain boolean, so the threshold defaults to
+ *     {@link DEFAULT_AUTODISTILL_THRESHOLD} (env-tunable via
+ *     `CREWHAUS_AUTODISTILL_THRESHOLD`).
+ *
+ *   - **Watermark** (`.crewhaus/feedback/.distill-state.json`): one small
+ *     JSON file holding the ISO timestamp of the newest feedback record
+ *     folded into the last auto-distill. "Unprocessed" = records with a
+ *     strictly greater `ts`. The trigger counts only unprocessed records,
+ *     but each distill re-reads EVERYTHING — the registry version is always
+ *     a full rebuild (merge semantics live in `mergeFeedback`), so the
+ *     watermark only rate-limits registrations, never shapes the dataset.
+ *     A failed registration leaves the watermark untouched (retried next
+ *     run); a distill that matched zero turns advances it (those records
+ *     are unmatchable — their sessions are gone — so retrying is noise).
+ *
+ *   - **Exit-rating prompt gating**: the pure decision function for the
+ *     one-keystroke `rate this session? [g]ood / [b]ad / [enter] skip`
+ *     prompt at clean REPL exit. NEVER prompts in non-TTY/piped mode; opt
+ *     out with CREWHAUS_NO_EXIT_RATING=1 or spec `feedback.exitPrompt:
+ *     false`. The raw-mode keystroke read stays in the entry file (thin
+ *     IO); everything decidable is here so it is unit-testable.
+ *
+ * Kept in a module with no import-time side effects (the CLI entry file
+ * runs an argv switch on import), mirroring `eval-history.ts` /
+ * `datasets.ts`. Filesystem access is limited to the watermark file.
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { DEFAULT_SPLIT_SPEC, registerDataset } from "./datasets";
+import { distill } from "./feedback";
+import { isRegistrySafeName } from "./regression-pin";
+/** Default "≥ N unprocessed ratings" trigger (the spec's `autoDistill` is a
+ *  plain boolean and carries no threshold of its own). */
+export const DEFAULT_AUTODISTILL_THRESHOLD = 5;
+/** Env override for the trigger threshold. */
+export const AUTODISTILL_THRESHOLD_ENV = "CREWHAUS_AUTODISTILL_THRESHOLD";
+/** Opt-out env for the REPL exit rating prompt. */
+export const NO_EXIT_RATING_ENV = "CREWHAUS_NO_EXIT_RATING";
+/** The watermark file, relative to the harness cwd. */
+export const DISTILL_STATE_RELPATH = join(".crewhaus", "feedback", ".distill-state.json");
+/** The registry dataset autoDistill maintains for a spec. */
+export function ratingsDatasetName(specName) {
+    return `${specName}-ratings`;
+}
+/** Narrow parsed JSON to a DistillState; undefined on any malformation (a
+ *  corrupt watermark degrades to "everything is unprocessed", never throws). */
+export function parseDistillState(text) {
+    let value;
+    try {
+        value = JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+    if (typeof value !== "object" || value === null)
+        return undefined;
+    const v = value;
+    if (v["schemaVersion"] !== 1)
+        return undefined;
+    if (typeof v["lastProcessedTs"] !== "string")
+        return undefined;
+    if (typeof v["processedCount"] !== "number")
+        return undefined;
+    return value;
+}
+export function readDistillState(path) {
+    if (!existsSync(path))
+        return undefined;
+    try {
+        return parseDistillState(readFileSync(path, "utf-8"));
+    }
+    catch {
+        return undefined;
+    }
+}
+export function writeDistillState(path, state) {
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 });
+}
+/** Records newer than the watermark (strict ISO-string comparison — the
+ *  capture surfaces all stamp `new Date().toISOString()`, which compares
+ *  lexicographically). No watermark → everything is unprocessed. */
+export function countUnprocessed(records, lastProcessedTs) {
+    if (lastProcessedTs === undefined)
+        return records.length;
+    return records.filter((r) => r.ts > lastProcessedTs).length;
+}
+export function newestTs(records) {
+    let max;
+    for (const r of records) {
+        if (max === undefined || r.ts > max)
+            max = r.ts;
+    }
+    return max;
+}
+// -------- trigger decision --------
+export function resolveAutoDistillThreshold(env) {
+    const raw = env[AUTODISTILL_THRESHOLD_ENV];
+    if (raw === undefined || raw === "")
+        return DEFAULT_AUTODISTILL_THRESHOLD;
+    const n = Number.parseInt(raw, 10);
+    if (Number.isNaN(n) || n < 1)
+        return DEFAULT_AUTODISTILL_THRESHOLD;
+    return n;
+}
+/** Whether the teardown should distill: the spec opted in
+ *  (`feedback.autoDistill: true`, block not disabled) AND enough
+ *  unprocessed ratings accumulated. */
+export function shouldAutoDistill(opts) {
+    if (opts.feedback === undefined) {
+        return { run: false, reason: "spec has no feedback block" };
+    }
+    if (opts.feedback.enabled === false) {
+        return { run: false, reason: "feedback block is disabled (enabled: false)" };
+    }
+    if (opts.feedback.autoDistill !== true) {
+        return { run: false, reason: "feedback.autoDistill is not enabled" };
+    }
+    if (opts.unprocessed < opts.threshold) {
+        return {
+            run: false,
+            reason: `${opts.unprocessed} unprocessed rating(s) < threshold ${opts.threshold}`,
+        };
+    }
+    return {
+        run: true,
+        reason: `${opts.unprocessed} unprocessed rating(s) ≥ threshold ${opts.threshold}`,
+    };
+}
+/**
+ * The teardown consumer. One printed line per outcome that did anything;
+ * silent when the trigger simply hasn't accumulated yet. Best-effort by
+ * contract — the CALLER wraps it so a failure can never turn a successful
+ * session into a non-zero exit.
+ */
+export async function maybeAutoDistill(opts) {
+    const write = opts.write ?? ((line) => process.stdout.write(`${line}\n`));
+    const env = opts.env ?? process.env;
+    const threshold = resolveAutoDistillThreshold(env);
+    const state = readDistillState(opts.stateFilePath);
+    const unprocessed = countUnprocessed(opts.records, state?.lastProcessedTs);
+    const decision = shouldAutoDistill({ feedback: opts.feedback, unprocessed, threshold });
+    if (!decision.run)
+        return { ran: false, reason: decision.reason };
+    const name = ratingsDatasetName(opts.specName);
+    if (!isRegistrySafeName(name)) {
+        write(`[feedback] auto-distill skipped: spec name "${opts.specName}" can't form a registry dataset name ("${name}")`);
+        return { ran: false, reason: "spec name is not registry-safe" };
+    }
+    const nowIso = (opts.now?.() ?? new Date()).toISOString();
+    const watermark = newestTs(opts.records) ?? nowIso;
+    // Full rebuild over EVERYTHING (see the module doc): merge semantics stay
+    // in mergeFeedback, and each registered version is self-contained.
+    const result = distill(opts.turns, opts.records, { minScore: opts.minScore ?? 0.7 });
+    if (result.samples.length === 0) {
+        // The ratings exist but none joined to a transcript turn (sessions
+        // purged/rotated). Advance the watermark so the same unmatchable
+        // records don't re-trigger a doomed distill on every run.
+        writeDistillState(opts.stateFilePath, {
+            schemaVersion: 1,
+            lastProcessedTs: watermark,
+            processedCount: opts.records.length,
+            ...(state?.lastRegistered !== undefined ? { lastRegistered: state.lastRegistered } : {}),
+        });
+        write("[feedback] auto-distill: no rated turn matched a transcript — nothing registered (watermark advanced)");
+        return { ran: false, reason: "no rated turns matched the transcripts" };
+    }
+    // Register FIRST, advance the watermark only on success — a transient
+    // registry failure retries on the next teardown instead of losing the
+    // trigger.
+    let record;
+    try {
+        record = await registerDataset({
+            registry: opts.registry,
+            name,
+            samples: result.samples,
+            splitSpec: DEFAULT_SPLIT_SPEC,
+        });
+    }
+    catch (err) {
+        write(`[feedback] auto-distill failed to register ${name}: ${err instanceof Error ? err.message : String(err)} (will retry next run)`);
+        return { ran: false, reason: "registry put failed" };
+    }
+    writeDistillState(opts.stateFilePath, {
+        schemaVersion: 1,
+        lastProcessedTs: watermark,
+        processedCount: opts.records.length,
+        lastRegistered: { name: record.name, version: record.version, at: nowIso },
+    });
+    write(`[feedback] auto-distilled ${result.stats.matchedTurns} rating(s) → registry:${record.name}@${record.version} ` +
+        `(train ${record.splits.train.length} / dev ${record.splits.dev.length} / test ${record.splits.test?.length ?? 0})`);
+    return {
+        ran: true,
+        reason: decision.reason,
+        registered: { name: record.name, version: record.version },
+    };
+}
+// -------- REPL exit rating prompt (gating + key parsing) --------
+export const EXIT_RATING_PROMPT = "rate this session? [g]ood / [b]ad / [enter] skip: ";
+export const EXIT_RATING_TIMEOUT_MS = 10_000;
+/** Turns with a non-empty assistant answer — the "session actually said
+ *  something worth rating" floor for the exit prompt. */
+export function countAssistantTurns(turns) {
+    return turns.filter((t) => t.output.trim() !== "").length;
+}
+/**
+ * The pure gate for the exit prompt. Prompts only when ALL hold: the spec
+ * has a feedback block (presence opts in), it isn't disabled
+ * (`enabled: false`) or prompt-opted-out (`exitPrompt: false`), the
+ * CREWHAUS_NO_EXIT_RATING env is not set, stdin is a real TTY (NEVER in
+ * piped/CI mode), and the session produced at least one assistant answer.
+ */
+export function shouldPromptExitRating(opts) {
+    if (opts.feedback === undefined)
+        return { prompt: false, reason: "spec has no feedback block" };
+    if (opts.feedback.enabled === false) {
+        return { prompt: false, reason: "feedback block is disabled (enabled: false)" };
+    }
+    if (opts.feedback.exitPrompt === false) {
+        return { prompt: false, reason: "feedback.exitPrompt is false" };
+    }
+    const optOut = opts.env[NO_EXIT_RATING_ENV];
+    if (optOut !== undefined && optOut !== "" && optOut !== "0") {
+        return { prompt: false, reason: `${NO_EXIT_RATING_ENV} is set` };
+    }
+    if (!opts.stdinIsTTY)
+        return { prompt: false, reason: "stdin is not a TTY" };
+    if (opts.assistantTurns < 1) {
+        return { prompt: false, reason: "session has no assistant turns to rate" };
+    }
+    return { prompt: true, reason: "feedback block present, TTY, rated-able session" };
+}
+/** Map the raw keystroke to a rating: g/G → good (thumbs up), b/B → bad
+ *  (thumbs down), anything else — enter, timeout (undefined), Ctrl-C/D —
+ *  skips. One keystroke, no confirmation, zero-cost to ignore. */
+export function parseExitRatingKey(key) {
+    if (key === undefined || key.length === 0)
+        return "skip";
+    const first = key[0];
+    if (first === "g" || first === "G")
+        return "up";
+    if (first === "b" || first === "B")
+        return "down";
+    return "skip";
+}