crewhaus 0.1.7 → 0.2.0

package/dist/fewshot.js

@@ -0,0 +1,158 @@
+/**
+ * Item #54 — harvest up-rated turns into a golden few-shot pool.
+ *
+ * `crewhaus fewshot harvest` mines POSITIVELY-rated conversation turns (the
+ * same `user_feedback` ratings `distill` consumes) into a curated pool of
+ * input→good-output few-shot examples. The pool is a versioned, provenance-
+ * tagged JSONL under `.crewhaus/fewshot/<spec>.jsonl`; `crewhaus optimize
+ * --few-shot <file>` injects the top-K examples into the agent prompt as
+ * in-context demonstrations, and `agent.instructions` can carry them at
+ * compile time via `formatFewShotForPrompt`.
+ *
+ * Everything here is pure + deterministic (stable ordering, id-based dedupe)
+ * so it is unit-testable; all filesystem access + the model/redactor wiring
+ * lives in `apps/cli/src/index.ts` (mirrors `feedback.ts` / `dataset-mine.ts`).
+ * The harvested OUTPUT text is the rated assistant answer (or the user's
+ * correction), so it is PII/secret-redacted before it lands in the pool: the
+ * caller threads a redactor built from `SYNTHESIZE_PII_DETECTORS` (the shared
+ * secret/API-key + DEFAULT_PII detector set) so a pasted credential never
+ * survives into the pool or the optimizer meta-prompt.
+ */
+import { mergeFeedback, normalizeRating } from "./feedback";
+/** Schema version of a persisted few-shot pool line. */
+export const FEWSHOT_SCHEMA_VERSION = 1;
+function turnKey(sessionId, turnNumber) {
+    return `${sessionId}#${turnNumber}`;
+}
+/**
+ * Harvest positively-rated turns into few-shot examples. A turn qualifies when
+ * its merged rating normalizes to ≥ `minScore`, OR it carries a `correction`
+ * (the user's own better answer — always gold). The example OUTPUT is the
+ * correction when present, else the assistant's answer; empty answers are
+ * skipped. Deterministic: examples are sorted by (score desc, id asc) and
+ * deduped by id, so re-harvesting the same feedback is idempotent.
+ */
+export async function harvestFewShot(turns, feedback, opts = {}) {
+    const minScore = opts.minScore ?? 0.7;
+    const redact = opts.redact ?? (async (t) => t);
+    const turnByKey = new Map();
+    for (const t of turns)
+        turnByKey.set(turnKey(t.sessionId, t.turnNumber), t);
+    const merged = mergeFeedback(feedback);
+    const byId = new Map();
+    let qualified = 0;
+    let skippedUnmatched = 0;
+    let skippedEmpty = 0;
+    for (const fb of merged) {
+        const turn = turnByKey.get(turnKey(fb.sessionId, fb.turnNumber));
+        if (turn === undefined) {
+            skippedUnmatched += 1;
+            continue;
+        }
+        const score = normalizeRating(fb);
+        const isCorrection = fb.correction !== undefined && fb.correction.trim() !== "";
+        const isPositive = (score !== undefined && score >= minScore) || isCorrection;
+        if (!isPositive)
+            continue;
+        const rawOutput = isCorrection ? fb.correction : turn.output;
+        if (rawOutput.trim() === "") {
+            skippedEmpty += 1;
+            continue;
+        }
+        const input = await redact(turn.input);
+        const output = await redact(rawOutput);
+        if (output.trim() === "") {
+            skippedEmpty += 1;
+            continue;
+        }
+        qualified += 1;
+        const id = `${fb.sessionId}_t${fb.turnNumber}`;
+        byId.set(id, {
+            schemaVersion: FEWSHOT_SCHEMA_VERSION,
+            id,
+            input,
+            output,
+            score: isCorrection ? 1 : (score ?? 1),
+            provenance: {
+                sessionId: fb.sessionId,
+                turnNumber: fb.turnNumber,
+                source: isCorrection ? "correction" : "rating",
+                ...(fb.rater !== undefined ? { rater: fb.rater } : {}),
+            },
+        });
+    }
+    const examples = [...byId.values()].sort((a, b) => b.score - a.score || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+    return {
+        examples,
+        stats: {
+            totalFeedback: merged.length,
+            qualified,
+            skippedUnmatched,
+            skippedEmpty,
+        },
+    };
+}
+/** Narrow an arbitrary parsed JSON value to a persisted FewShotExample. */
+export function isFewShotExample(value) {
+    if (typeof value !== "object" || value === null)
+        return false;
+    const v = value;
+    if (v["schemaVersion"] !== 1)
+        return false;
+    if (typeof v["id"] !== "string")
+        return false;
+    if (typeof v["input"] !== "string" || typeof v["output"] !== "string")
+        return false;
+    if (typeof v["score"] !== "number" || !Number.isFinite(v["score"]))
+        return false;
+    const p = v["provenance"];
+    if (typeof p !== "object" || p === null)
+        return false;
+    const pr = p;
+    return typeof pr["sessionId"] === "string" && typeof pr["turnNumber"] === "number";
+}
+/**
+ * Merge a freshly-harvested pool into an existing one, deduping by id (the
+ * newer example wins so re-runs refresh outputs without duplicating). Returns
+ * the deterministically-ordered union.
+ */
+export function mergePools(existing, fresh) {
+    const byId = new Map();
+    for (const e of existing)
+        byId.set(e.id, e);
+    for (const e of fresh)
+        byId.set(e.id, e);
+    return [...byId.values()].sort((a, b) => b.score - a.score || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+}
+/** Serialize a pool to JSONL (one example per line). */
+export function poolToJsonl(examples) {
+    return `${examples.map((e) => JSON.stringify(e)).join("\n")}\n`;
+}
+const FEWSHOT_CLOSE_TAG_RE = /<\s*\/\s*few_shot_examples\s*>/gi;
+/**
+ * #54 F5 — delimiter safety. An example's input/output is user/model text and
+ * can be adversarially shaped, so before interpolating it into the
+ * `<few_shot_examples>` block we neutralize any embedded closing tag: a
+ * `</few_shot_examples>` inside an example would otherwise let a poisoned
+ * example terminate the block early and inject trailing instructions. The
+ * closing tag is rewritten to an inert `<\/few_shot_examples>` (content
+ * preserved for the model, but no longer parses as the real delimiter).
+ * Redaction of secrets/PII already runs upstream — this is purely structural.
+ */
+function escapeFewShotContent(text) {
+    return text.replace(FEWSHOT_CLOSE_TAG_RE, "<\\/few_shot_examples>");
+}
+/**
+ * Render the top-K examples as an in-context few-shot block for the system
+ * prompt / `agent.instructions`. Empty input → "" so callers append
+ * unconditionally. `k` caps how many examples are injected (default 5).
+ */
+export function formatFewShotForPrompt(examples, k = 5) {
+    const top = examples.slice(0, Math.max(0, k));
+    if (top.length === 0)
+        return "";
+    const blocks = top
+        .map((e, i) => `Example ${i + 1}:\nUser: ${escapeFewShotContent(e.input)}\nAssistant: ${escapeFewShotContent(e.output)}`)
+        .join("\n\n");
+    return `<few_shot_examples>\nThese are examples of responses users rated highly. Follow their style and quality:\n\n${blocks}\n</few_shot_examples>`;
+}

package/dist/fleet.d.ts

@@ -0,0 +1,207 @@
+import type { Manifest } from "@crewhaus/spec-registry";
+/** The spec file that roots a standalone harness. */
+export declare const HARNESS_SPEC_FILENAME = "crewhaus.yaml";
+/** Thrown for operational failures (missing root, empty filter, disallowed
+ *  bulk subcommand). The CLI entry file catches it and routes the message
+ *  through `die()`; tests assert on `.message` without the process exiting. */
+export declare class FleetError extends Error {
+    readonly name = "FleetError";
+}
+/** A discovered harness: its absolute directory + the spec path within it. */
+export type DiscoveredHarness = {
+    readonly dir: string;
+    readonly specPath: string;
+};
+/**
+ * Walk `root` for directories carrying a `crewhaus.yaml`, bounded to
+ * {@link MAX_DISCOVERY_DEPTH}. Never descends into `.crewhaus`, `node_modules`,
+ * `.git`, `dist`, or `.worktrees`. A directory that IS a harness is still
+ * descended into (a harness may nest sub-harnesses in a demo tree), just not
+ * through its state dir. Results are sorted by directory for stable output.
+ */
+export declare function discoverHarnesses(root: string): DiscoveredHarness[];
+/** The identity fields a fleet row needs, read best-effort. */
+export type SpecHeader = {
+    readonly name?: string;
+    readonly target?: string;
+    readonly model?: string;
+};
+/**
+ * Read `name`, `target`, and `model` from a `crewhaus.yaml` WITHOUT a full
+ * schema parse. A fleet scan must survive a stale/invalid spec (schema drift,
+ * a half-edited file), so this is a tolerant top-level-key scrape rather than
+ * `parseSpec`. `name`/`target` are top-level on every target shape; `model`
+ * lives at `agent.model` (cli/channel/voice) OR top-level `model`
+ * (workflow/graph/pipeline/crew) depending on the shape — the first
+ * `model:` at any indentation is taken. Values are unquoted and trimmed.
+ */
+export declare function readSpecHeader(yamlText: string): SpecHeader;
+/** The `.crewhaus/specs`-registered version + env pins for a harness spec. */
+export type RegistryStatus = {
+    /** Latest registered version (`v3`), or undefined when nothing is registered. */
+    readonly latestVersion?: string;
+    /** env → version pins from the registry manifest. */
+    readonly pins: Readonly<Record<string, string>>;
+};
+/** The newest recorded eval run for a harness. */
+export type LastEval = {
+    readonly datasetName: string;
+    readonly passRate: number;
+    readonly ts: string;
+};
+/** Minimal shape of an eval index entry the fleet row needs. */
+export type LastEvalEntry = {
+    readonly datasetName: string;
+    readonly passRate: number;
+    readonly ts: string;
+};
+/** One harness's rolled-up inventory row. */
+export type HarnessInventory = {
+    readonly dir: string;
+    readonly header: SpecHeader;
+    /** Best-effort spec name (header name, else the dir basename). */
+    readonly specName: string;
+    readonly registry: RegistryStatus;
+    readonly lastEval?: LastEval;
+    readonly sessionCount: number;
+    readonly feedbackCount: number;
+    /** True when the spec file did not read at all (unreadable/absent). */
+    readonly specUnreadable: boolean;
+};
+/** Count `sess_<16hex>.json` files under `.crewhaus/sessions`. */
+export declare function countSessions(harnessDir: string): number;
+/**
+ * Distinct feedback a harness carries, counted the way `crewhaus distill`
+ * folds it: `user_feedback` events in `sessions/*.jsonl` PLUS bare records in
+ * `feedback/*.jsonl` (`extractFeedbackRecords` accepts both encodings), then
+ * `mergeFeedback` collapses to one per (sessionId, turnNumber). The row shows
+ * distinct rated turns, not raw line count, so a re-rated turn counts once.
+ */
+export declare function countFeedback(harnessDir: string): number;
+/** The newest run in an eval index, or undefined. Newest by `ts` (ISO-8601
+ *  lexical order is chronological). */
+export declare function lastEvalFor(evalEntries: ReadonlyArray<LastEvalEntry>): LastEval | undefined;
+/**
+ * Manifest reader seam — production passes a closure over
+ * `@crewhaus/spec-registry`; tests pass a stub. Returns undefined when the
+ * spec has no registry entry (never registered / compiled).
+ */
+export type ManifestReader = (specName: string, registryRoot: string) => Promise<Manifest | undefined>;
+/** Eval-index reader seam — production passes `readRunIndex`; tests stub. */
+export type EvalIndexReader = (evalsDir: string) => ReadonlyArray<LastEvalEntry>;
+export type BuildInventoryDeps = {
+    readonly readManifest: ManifestReader;
+    readonly readEvalIndex: EvalIndexReader;
+};
+/**
+ * Aggregate one harness directory into an inventory row. All reads are
+ * tolerant: a missing state dir, an unregistered spec, or an unreadable spec
+ * file each degrade to an empty/absent field rather than throwing, so one
+ * broken harness never aborts a `fleet list`.
+ */
+export declare function buildHarnessInventory(harness: DiscoveredHarness, deps: BuildInventoryDeps): Promise<HarnessInventory>;
+/** Build the full fleet inventory for a root. */
+export declare function buildFleetInventory(root: string, deps: BuildInventoryDeps): Promise<HarnessInventory[]>;
+/** Render the inventory as aligned columns for `fleet list`. */
+export declare function formatInventory(rows: ReadonlyArray<HarnessInventory>, root: string): ReadonlyArray<string>;
+export type HarnessHealth = {
+    readonly dir: string;
+    readonly specName: string;
+    /** True when a compiled bundle / registry entry exists (deployable). */
+    readonly registered: boolean;
+    /** env pins present. */
+    readonly pinnedEnvs: ReadonlyArray<string>;
+    /** True when a pinned eval baseline exists AND the last run met/held it. */
+    readonly evalHealthy: boolean;
+    readonly evalNote: string;
+    /** Open incidents from the ops-batch incidents dir, if present. */
+    readonly openIncidents: number;
+    /** Whether `.crewhaus/audit` exists (tamper-evidence available). */
+    readonly hasAudit: boolean;
+};
+/** Reader seam: newest pass-rate + whether it dropped vs the pinned baseline. */
+export type EvalHealthReader = (evalsDir: string, specName: string) => {
+    healthy: boolean;
+    note: string;
+};
+/** Count open incidents under `.crewhaus/incidents` (files not ending
+ *  `.resolved.json`). Absent dir → 0. The ops batch writes one JSON per
+ *  incident; a resolved one is renamed `<id>.resolved.json`. */
+export declare function countOpenIncidents(harnessDir: string): number;
+export declare function buildHarnessHealth(inv: HarnessInventory, readEvalHealth: EvalHealthReader): Promise<HarnessHealth>;
+/** Render the health rollup as status lines for `fleet status`. */
+export declare function formatHealth(rows: ReadonlyArray<HarnessHealth>, root: string): ReadonlyArray<string>;
+/** ✓ healthy / ✗ needs attention. Attention = an open incident, an eval
+ *  regression, or an unregistered harness that nonetheless has env pins
+ *  (a pin pointing at nothing registered). */
+export declare function healthMark(r: HarnessHealth): "✓" | "✗";
+/** Read-only / idempotent subcommands the bulk runner accepts without an
+ *  explicit opt-in. Each maps to a `crewhaus <argv...>` invocation. */
+export declare const READ_ONLY_BULK_COMMANDS: Readonly<Record<string, ReadonlyArray<string>>>;
+/** A resolved bulk-run plan: the base argv + whether it is a mutating command. */
+export type BulkCommandPlan = {
+    /** The base argv (e.g. ["security","digest"]). */
+    readonly argv: ReadonlyArray<string>;
+    /** True when the command is off the read-only allow-list. */
+    readonly mutating: boolean;
+};
+/**
+ * Resolve a `fleet run` subcommand + trailing args into a plan. Refuses
+ * anything off the read-only allow-list unless `allowMutating` is set. The
+ * longest allow-listed prefix wins, so `security digest` resolves ahead of a
+ * bare `security`.
+ */
+export declare function resolveBulkCommand(subcommandTokens: ReadonlyArray<string>, allowMutating: boolean): BulkCommandPlan;
+/** Match a harness against a `--filter` glob, tested against BOTH the spec
+ *  name and the directory basename, so `--filter '*bot*'` catches either. */
+export declare function matchesFilter(inv: HarnessInventory, filter: string | undefined): boolean;
+/** Launch seam: run `crewhaus <argv>` in `cwd`, resolve to an exit code + the
+ *  captured tail of output. Production wires this to `Bun.spawn`; tests inject
+ *  a deterministic stub so no subprocess runs. */
+export type FleetRunner = (opts: {
+    readonly cwd: string;
+    readonly argv: ReadonlyArray<string>;
+}) => Promise<{
+    readonly exitCode: number;
+    readonly tail: string;
+}>;
+/** Per-harness confirm seam for `--allow-mutating` (interactive prompt in the
+ *  CLI; a predicate in tests). Returning false skips that harness. */
+export type ConfirmMutating = (inv: HarnessInventory, argv: ReadonlyArray<string>) => Promise<boolean>;
+export type BulkRunResult = {
+    readonly inv: HarnessInventory;
+    readonly ran: boolean;
+    /** exit code when ran; undefined when skipped (filter/decline). */
+    readonly exitCode?: number;
+    readonly tail?: string;
+    /** Reason a harness was skipped, if it was. */
+    readonly skipReason?: string;
+};
+export type RunFleetBulkOptions = {
+    readonly root: string;
+    readonly subcommandTokens: ReadonlyArray<string>;
+    readonly filter?: string;
+    readonly allowMutating: boolean;
+    readonly deps: BuildInventoryDeps;
+    readonly runner: FleetRunner;
+    readonly confirm: ConfirmMutating;
+};
+export type RunFleetBulkReport = {
+    readonly plan: BulkCommandPlan;
+    readonly results: ReadonlyArray<BulkRunResult>;
+    /** Harnesses that ran and exited non-zero. */
+    readonly failed: number;
+    /** Harnesses that ran and exited zero. */
+    readonly passed: number;
+    /** Harnesses skipped (filtered out or declined). */
+    readonly skipped: number;
+};
+/**
+ * Run one subcommand across the filtered fleet. Read-only commands run
+ * unconditionally on every matched harness; a mutating command (only reachable
+ * with `allowMutating`) runs a harness only when `confirm` approves it. The
+ * runner is the sole side-effect surface, injected for testability.
+ */
+export declare function runFleetBulk(opts: RunFleetBulkOptions): Promise<RunFleetBulkReport>;
+/** Render a bulk-run report as summary lines for `fleet run`. */
+export declare function formatBulkReport(report: RunFleetBulkReport): ReadonlyArray<string>;