crewhaus 0.1.8 → 0.2.0

package/dist/egress-triage.d.ts

@@ -0,0 +1,121 @@
+import type { Spec } from "@crewhaus/spec";
+import type { AdviceFinding } from "./advise-rules";
+/**
+ * AUTOMATION-OPPORTUNITIES.md item 20 — `crewhaus egress review [--propose]`:
+ * triage durable egress/boundary history into learned security spec
+ * suggestions. Side-effect-free so it is unit-testable, mirroring
+ * `advise-rules.ts` (all filesystem access stays in `index.ts`).
+ *
+ * INPUT (durability, verified 2026-07): egress verdicts are NOW durable —
+ * runtime-core's `egressAuditSink` (this branch) appends one `egress_decision`
+ * record per NON-PASS verdict to the hash-chained `.crewhaus/audit`, payload
+ * `{ sinkId, sinkScope, verdict: "warn"|"block", originsFound[], matchCount }`
+ * (lineage summary only, never the raw payload). Rule-based justification
+ * denials come from the `permission_justification_evaluated` records the
+ * intent gate already writes.
+ *
+ * OUTPUT: three proposal families, matching the item:
+ *
+ *   1. Per-sink RELAXATION — sinks that warned ≥ warnRelaxMin times with ZERO
+ *      blocks and a single stable origin: the flow is chronic-but-benign, so
+ *      the operator may want to whitelist it. EMITTED AS ADVICE TEXT, never a
+ *      patch: the natural home is a `security.egressPolicy` block, but that
+ *      schema field is RESERVED for the egress-fabric FRs (FR-002/006) — the
+ *      spec schema intentionally does NOT carry it today, and
+ *      `OPTIMIZABLE_PATHS` lists `["security","egressPolicy"]` on their behalf.
+ *      Adding the field here would clobber their ownership, so we coordinate:
+ *      the proposal is advice text pointing at where the relaxation will live
+ *      once that block ships. (See packages/spec/src/index.ts securityBlock
+ *      doc comment: "Do NOT add egressPolicy here".)
+ *
+ *   2. `security.egressMatcher: semantic` — when warn-noise is HIGH but blocks
+ *      are ZERO across sinks, substring matching is over-firing on benign
+ *      lineage overlaps; the embedding matcher scores by similarity instead.
+ *      ADVICE-ONLY: `["security","egressMatcher"]` is NOT in OPTIMIZABLE_PATHS
+ *      (switching the matcher changes detection semantics — a human decision),
+ *      so it must never ride `optimize --from-advice`.
+ *
+ *   3. `security.justification.judge: claude` — when RULE-BASED justification
+ *      denials are frequent, the deterministic judge is likely over-denying;
+ *      the model-backed judge reasons about intent. `["security","justification"]`
+ *      IS whitelisted (cli/managed), so this rides the eval-gated
+ *      `optimize --from-advice` apply as a SPEC-PATCH when a spec is present
+ *      and the patch validates; otherwise it degrades to advice text.
+ */
+export type EgressDecisionRecord = {
+    readonly sinkId: string;
+    readonly sinkScope: string;
+    readonly verdict: "warn" | "block";
+    readonly originsFound: ReadonlyArray<string>;
+    readonly matchCount: number;
+};
+/** Per (sink, origin, scope) cluster of egress records. */
+export type EgressCluster = {
+    readonly sinkId: string;
+    readonly sinkScope: string;
+    /** Origins observed across this sink's non-pass records, deduped + sorted. */
+    readonly origins: ReadonlyArray<string>;
+    readonly warned: number;
+    readonly blocked: number;
+    /** Total records folded into this cluster. */
+    readonly total: number;
+};
+export type EgressTriageContext = {
+    /** Non-pass egress clusters keyed by sinkId, ranked by (blocked, warned). */
+    readonly clusters: ReadonlyArray<EgressCluster>;
+    /** Total warn / block records across all sinks (the matcher-noise signal). */
+    readonly totalWarned: number;
+    readonly totalBlocked: number;
+    /** Rule-based justification denials — the judge-upgrade signal. */
+    readonly ruleBasedDenials: number;
+    readonly ruleBasedEvaluated: number;
+    /** Egress records parsed; malformed/other kinds skipped. */
+    readonly egressRecords: number;
+};
+/**
+ * Fold parsed `.crewhaus/audit` records into the triage context. Tolerant of
+ * malformed/unknown records (skipped). Reads two kinds:
+ *   - `egress_decision` → per-sink warn/block clusters + totals
+ *   - `permission_justification_evaluated` → rule-based deny counts
+ */
+export declare function buildEgressTriageContext(auditRecords: ReadonlyArray<unknown>): EgressTriageContext;
+export type EgressTriageThresholds = {
+    /** Warns on ONE sink (with 0 blocks) at/above which a relaxation is proposed. */
+    readonly warnRelaxMin: number;
+    /** Total warns (with 0 total blocks) at/above which semantic is proposed. */
+    readonly semanticWarnMin: number;
+    /** Rule-based denials at/above which the claude-judge upgrade is proposed. */
+    readonly judgeDenialMin: number;
+};
+export declare const DEFAULT_EGRESS_TRIAGE_THRESHOLDS: EgressTriageThresholds;
+/** Targets whose schema carries `security.justification` AND for which
+ *  `["security","justification"]` is optimizable (rides --from-advice). */
+export declare const JUSTIFICATION_PATCH_TARGETS: ReadonlySet<string>;
+export type EgressTriageOptions = {
+    readonly spec?: Spec;
+    readonly thresholds?: Partial<EgressTriageThresholds>;
+};
+/**
+ * Proposal 1 — per-sink relaxation for chronic warn-only flows. ADVICE-ONLY:
+ * the relaxation's home (`security.egressPolicy`) is reserved for the egress
+ * FRs and does not exist in the schema yet, so we emit guidance rather than a
+ * schema-adding patch (coordinate, don't clobber).
+ */
+export declare function ruleSinkRelaxation(ctx: EgressTriageContext, opts?: EgressTriageOptions): AdviceFinding[];
+/**
+ * Proposal 2 — `security.egressMatcher: semantic` when warn-noise is high but
+ * blocks are zero (the substring matcher is over-firing on benign lineage
+ * overlaps). ADVICE-ONLY: egressMatcher is not optimizer-whitelisted.
+ */
+export declare function ruleSemanticMatcher(ctx: EgressTriageContext, opts?: EgressTriageOptions): AdviceFinding[];
+/**
+ * Proposal 3 — `security.justification.judge: claude` when rule-based denials
+ * are frequent. SPEC-PATCH when the spec is a justification-optimizable target
+ * and the patch validates (rides `optimize --from-advice`); else advice.
+ */
+export declare function ruleJustificationJudgeUpgrade(ctx: EgressTriageContext, opts?: EgressTriageOptions): AdviceFinding[];
+export declare const EGRESS_TRIAGE_RULES: readonly [typeof ruleSinkRelaxation, typeof ruleSemanticMatcher, typeof ruleJustificationJudgeUpgrade];
+/** Run every triage rule and rank: warn before info, then by primary count. */
+export declare function runEgressTriage(ctx: EgressTriageContext, opts?: EgressTriageOptions): AdviceFinding[];
+/** One line per finding for the CLI's text mode (mirrors formatFindingLines). */
+export declare function formatEgressFindingLines(f: AdviceFinding): string[];

package/dist/egress-triage.js

@@ -0,0 +1,261 @@
+import { validatePatch } from "@crewhaus/spec-patch";
+function asObject(v) {
+    return v !== null && typeof v === "object" && !Array.isArray(v)
+        ? v
+        : undefined;
+}
+/**
+ * Fold parsed `.crewhaus/audit` records into the triage context. Tolerant of
+ * malformed/unknown records (skipped). Reads two kinds:
+ *   - `egress_decision` → per-sink warn/block clusters + totals
+ *   - `permission_justification_evaluated` → rule-based deny counts
+ */
+export function buildEgressTriageContext(auditRecords) {
+    const bySink = new Map();
+    let totalWarned = 0;
+    let totalBlocked = 0;
+    let ruleBasedDenials = 0;
+    let ruleBasedEvaluated = 0;
+    let egressRecords = 0;
+    for (const rec of auditRecords) {
+        const obj = asObject(rec);
+        if (obj === undefined)
+            continue;
+        const kind = obj["kind"];
+        if (kind === "egress_decision") {
+            const p = asObject(obj["payload"]);
+            if (p === undefined)
+                continue;
+            const verdict = p["verdict"];
+            if (verdict !== "warn" && verdict !== "block")
+                continue;
+            egressRecords += 1;
+            const sinkId = typeof p["sinkId"] === "string" ? p["sinkId"] : "(unknown sink)";
+            const scope = typeof p["sinkScope"] === "string" ? p["sinkScope"] : "(unknown scope)";
+            const acc = bySink.get(sinkId) ?? {
+                scope,
+                origins: new Set(),
+                warned: 0,
+                blocked: 0,
+                total: 0,
+            };
+            if (verdict === "warn") {
+                acc.warned += 1;
+                totalWarned += 1;
+            }
+            else {
+                acc.blocked += 1;
+                totalBlocked += 1;
+            }
+            acc.total += 1;
+            if (Array.isArray(p["originsFound"])) {
+                for (const o of p["originsFound"])
+                    if (typeof o === "string")
+                        acc.origins.add(o);
+            }
+            bySink.set(sinkId, acc);
+            continue;
+        }
+        if (kind === "permission_justification_evaluated") {
+            const p = asObject(obj["payload"]);
+            if (p === undefined)
+                continue;
+            // The rule-based judge stamps judgeModel "rule-based" (justification-gate).
+            if (p["judgeModel"] !== "rule-based")
+                continue;
+            ruleBasedEvaluated += 1;
+            if (p["verdict"] === "deny")
+                ruleBasedDenials += 1;
+        }
+    }
+    const clusters = [...bySink.entries()]
+        .map(([sinkId, a]) => ({
+        sinkId,
+        sinkScope: a.scope,
+        origins: [...a.origins].sort(),
+        warned: a.warned,
+        blocked: a.blocked,
+        total: a.total,
+    }))
+        .sort((x, y) => y.blocked - x.blocked || y.warned - x.warned || x.sinkId.localeCompare(y.sinkId));
+    return {
+        clusters,
+        totalWarned,
+        totalBlocked,
+        ruleBasedDenials,
+        ruleBasedEvaluated,
+        egressRecords,
+    };
+}
+export const DEFAULT_EGRESS_TRIAGE_THRESHOLDS = {
+    warnRelaxMin: 5,
+    semanticWarnMin: 10,
+    judgeDenialMin: 5,
+};
+/** Targets whose schema carries `security.justification` AND for which
+ *  `["security","justification"]` is optimizable (rides --from-advice). */
+export const JUSTIFICATION_PATCH_TARGETS = new Set(["cli", "managed"]);
+// -------- helper: patch-or-advice (mirrors advise-rules) --------
+function patchOrAdvice(spec, patch, fallback) {
+    if (spec === undefined)
+        return { kind: "advice", text: fallback };
+    try {
+        validatePatch(spec, patch);
+        return { kind: "spec-patch", patch };
+    }
+    catch {
+        return { kind: "advice", text: fallback };
+    }
+}
+function specJustificationJudge(spec) {
+    if (spec === undefined)
+        return undefined;
+    const security = spec["security"];
+    const just = asObject(security)?.["justification"];
+    const judge = asObject(just)?.["judge"];
+    return typeof judge === "string" ? judge : undefined;
+}
+function resolveThresholds(opts) {
+    return { ...DEFAULT_EGRESS_TRIAGE_THRESHOLDS, ...(opts?.thresholds ?? {}) };
+}
+/**
+ * Proposal 1 — per-sink relaxation for chronic warn-only flows. ADVICE-ONLY:
+ * the relaxation's home (`security.egressPolicy`) is reserved for the egress
+ * FRs and does not exist in the schema yet, so we emit guidance rather than a
+ * schema-adding patch (coordinate, don't clobber).
+ */
+export function ruleSinkRelaxation(ctx, opts) {
+    const t = resolveThresholds(opts);
+    const findings = [];
+    for (const c of ctx.clusters) {
+        if (c.blocked > 0)
+            continue; // never relax a sink that also blocked
+        if (c.warned < t.warnRelaxMin)
+            continue;
+        const originList = c.origins.length > 0 ? c.origins.join(", ") : "(unknown)";
+        findings.push({
+            id: `egress-relax:${c.sinkId}`,
+            severity: "info",
+            summary: `sink ${c.sinkId} warned ${c.warned}× with 0 blocks (origins: ${originList})`,
+            evidence: [
+                `${c.warned} warn-tier egress decisions to ${c.sinkId} (scope ${c.sinkScope}), 0 blocks (threshold: ≥${t.warnRelaxMin} warns)`,
+                `origins seen: ${originList}`,
+            ],
+            counts: { warned: c.warned, blocked: c.blocked },
+            suggestion: {
+                kind: "advice",
+                text: `The sink \`${c.sinkId}\` has warned repeatedly (${c.warned}×) carrying content from [${originList}] but never had to BLOCK — a chronic-but-benign cross-origin flow. ` +
+                    `If this flow is intended, relax it per-sink once the \`security.egressPolicy\` block ships (reserved for the egress-fabric FRs; not yet in the spec schema): a per-sink override like \`${c.sinkId}: { ${c.origins[0] ?? "origin"}: pass }\`. ` +
+                    `Until then, review whether \`${c.sinkId}\` should carry [${originList}] content at all — a durable warn stream is exactly the signal to audit before whitelisting. This is deliberately NOT auto-applied.`,
+            },
+        });
+    }
+    return findings;
+}
+/**
+ * Proposal 2 — `security.egressMatcher: semantic` when warn-noise is high but
+ * blocks are zero (the substring matcher is over-firing on benign lineage
+ * overlaps). ADVICE-ONLY: egressMatcher is not optimizer-whitelisted.
+ */
+export function ruleSemanticMatcher(ctx, opts) {
+    const t = resolveThresholds(opts);
+    if (ctx.totalBlocked > 0)
+        return []; // real blocks → don't loosen matching
+    if (ctx.totalWarned < t.semanticWarnMin)
+        return [];
+    const current = asObject(opts?.spec?.["security"])?.["egressMatcher"];
+    if (current === "semantic")
+        return []; // already on
+    return [
+        {
+            id: "egress-matcher-semantic",
+            severity: "warn",
+            summary: `${ctx.totalWarned} warn-tier egress decisions, 0 blocks — substring matcher likely over-firing`,
+            evidence: [
+                `${ctx.totalWarned} warns across ${ctx.clusters.length} sink(s), 0 blocks (threshold: ≥${t.semanticWarnMin} warns, 0 blocks)`,
+                "high warn-noise with zero blocks is the signature of a substring matcher hitting benign lineage overlaps",
+            ],
+            counts: { warned: ctx.totalWarned, blocked: ctx.totalBlocked },
+            suggestion: {
+                kind: "advice",
+                text: "Set `security.egressMatcher: semantic` in the spec so outbound payloads are scored against tagged data-lineage by cosine similarity instead of verbatim substring — this cuts the false-positive warn stream while keeping the same per-origin/per-sink block policy. Switching matchers changes detection semantics, so it is a human decision (never auto-applied) and needs an embedder wired (`--egress-embedder`).",
+            },
+        },
+    ];
+}
+/**
+ * Proposal 3 — `security.justification.judge: claude` when rule-based denials
+ * are frequent. SPEC-PATCH when the spec is a justification-optimizable target
+ * and the patch validates (rides `optimize --from-advice`); else advice.
+ */
+export function ruleJustificationJudgeUpgrade(ctx, opts) {
+    const t = resolveThresholds(opts);
+    if (ctx.ruleBasedDenials < t.judgeDenialMin)
+        return [];
+    const spec = opts?.spec;
+    if (specJustificationJudge(spec) === "claude")
+        return []; // already upgraded
+    const denyRate = ctx.ruleBasedEvaluated > 0 ? ctx.ruleBasedDenials / ctx.ruleBasedEvaluated : null;
+    const adviceText = "The rule-based justification judge denied frequently — the deterministic judge is prone to over-denying legitimate calls whose justification does not lexically match the goal. Set `security.justification.judge: claude` (and optionally `security.justification.model`) so the model-backed judge reasons about intent. This is eval-gated: `optimize --from-advice` applies it only if the eval pass-rate holds.";
+    // The whole `security.justification` block is the optimizable path; compose
+    // it preserving any existing model selection.
+    const existingModel = asObject(asObject(spec?.["security"])?.["justification"])?.["model"];
+    const value = { judge: "claude" };
+    if (typeof existingModel === "string")
+        value["model"] = existingModel;
+    const patch = {
+        target: spec?.target ?? "cli",
+        path: ["security", "justification"],
+        op: specJustificationJudge(spec) !== undefined ? "replace" : "add",
+        value,
+        rationale: `egress review: ${ctx.ruleBasedDenials} rule-based justification denials — upgrade to the claude judge`,
+    };
+    // Gate the patch on a justification-optimizable target; otherwise advice.
+    const suggestion = spec !== undefined && JUSTIFICATION_PATCH_TARGETS.has(spec.target)
+        ? patchOrAdvice(spec, patch, adviceText)
+        : { kind: "advice", text: adviceText };
+    return [
+        {
+            id: "justification-judge-upgrade",
+            severity: "warn",
+            summary: `${ctx.ruleBasedDenials} rule-based justification denials — consider the claude judge`,
+            evidence: [
+                `${ctx.ruleBasedDenials} of ${ctx.ruleBasedEvaluated} rule-based justification evaluations denied${denyRate !== null ? ` (${(denyRate * 100).toFixed(0)}% deny rate)` : ""} (threshold: ≥${t.judgeDenialMin} denials)`,
+            ],
+            counts: { denials: ctx.ruleBasedDenials, evaluated: ctx.ruleBasedEvaluated },
+            suggestion,
+        },
+    ];
+}
+export const EGRESS_TRIAGE_RULES = [
+    ruleSinkRelaxation,
+    ruleSemanticMatcher,
+    ruleJustificationJudgeUpgrade,
+];
+/** Run every triage rule and rank: warn before info, then by primary count. */
+export function runEgressTriage(ctx, opts) {
+    const findings = EGRESS_TRIAGE_RULES.flatMap((rule) => rule(ctx, opts));
+    const magnitude = (f) => Object.values(f.counts).reduce((max, n) => Math.max(max, n), 0);
+    return findings.sort((a, b) => {
+        if (a.severity !== b.severity)
+            return a.severity === "warn" ? -1 : 1;
+        const diff = magnitude(b) - magnitude(a);
+        if (diff !== 0)
+            return diff;
+        return a.id.localeCompare(b.id);
+    });
+}
+/** One line per finding for the CLI's text mode (mirrors formatFindingLines). */
+export function formatEgressFindingLines(f) {
+    const lines = [`[${f.severity}] ${f.id} — ${f.summary}`];
+    for (const e of f.evidence)
+        lines.push(`  · ${e}`);
+    if (f.suggestion.kind === "spec-patch") {
+        const p = f.suggestion.patch;
+        lines.push(`  patch: ${p.op} ${p.path.join(".")} → ${JSON.stringify(p.value)}`);
+    }
+    else {
+        lines.push(`  advice: ${f.suggestion.text}`);
+    }
+    return lines;
+}

package/dist/eval-bridge.d.ts

@@ -0,0 +1,114 @@
+/**
+ * Item 10 — `crewhaus compile --with-eval-harness`: auto-generate an eval
+ * bridge (a `target: eval` bundle) for ANY shape's spec.
+ *
+ * `crewhaus eval` / `optimize` / `flywheel` hard-error on non-cli targets, so
+ * the ~13 non-cli shapes (channel-bot, voice, managed, onchain, …) cannot
+ * consume their own distilled Slack/web-UI feedback through the eval loop.
+ * The eval bridge closes that gap: it PROJECTS a shape's lowered IR into a
+ * sibling `IrEvalV0` — reusing the SAME agent model/instructions/tools — so
+ * `target-eval-bundle` can emit a first-class eval bundle that drives the
+ * shape's own datasets + graders.
+ *
+ * Two pieces live here, both side-effect-free (the CLI entry file does the IO
+ * — `mkdirSync`/`writeFileSync` of the emitted bundle):
+ *
+ *   1. `projectEvalIr(ir, opts)` — the IR → IrEvalV0 projection. Variant-
+ *      agnostic: it pulls `{ model, instructions, tools }` off whichever agent
+ *      sub-shape the source variant carries (cli/channel/managed/voice/browser/
+ *      onchain/onchain-game/batch/research all carry a top-level `agent`).
+ *      Shapes with NO single agent (workflow/graph/crew/pipeline are
+ *      multi-stage) throw a clear diagnostic — their eval bridge is a per-step
+ *      concern, deferred.
+ *
+ *   2. `selectInvoker(target)` — names the per-shape AgentInvoker strategy the
+ *      eval run should wrap the shape's runtime with (via the eval-runner
+ *      `opts.invoker` seam). The emitted bundle drives the default single-turn
+ *      `runChatLoop` invoker (correct for the one-shot shapes); this mapping
+ *      records the runtime seam each shape's eval should ideally wrap so the
+ *      bundle README + the CLI report point the author at it. Deterministic and
+ *      exhaustive over the shape set.
+ */
+import type { IrNode } from "@crewhaus/ir";
+/** Thrown on an un-projectable shape / malformed bridge options. The CLI entry
+ *  file routes it through `die()`; tests assert on `.message`. */
+export declare class EvalBridgeError extends Error {
+    readonly name = "EvalBridgeError";
+}
+/** The projected eval IR — structurally an `IrEvalV0` (target: "eval"). Kept as
+ *  a local type so this module needs no value import from `@crewhaus/ir`. */
+export type ProjectedEvalIr = {
+    readonly version: 0;
+    readonly name: string;
+    readonly target: "eval";
+    readonly agent: {
+        readonly model: string;
+        readonly instructions: string;
+        readonly tools: readonly string[];
+    };
+    readonly dataset: {
+        readonly name: string;
+        readonly version: string;
+        readonly split: "train" | "dev" | "test";
+    };
+    readonly graders: readonly {
+        readonly name: string;
+        readonly opts?: Record<string, unknown>;
+    }[];
+    readonly concurrency: number;
+    readonly seed?: number;
+};
+/** The per-shape invoker strategy an eval run should wrap the shape with. Each
+ *  entry names the runtime seam the shape resumes/steps through so a downstream
+ *  invoker (via eval-runner `opts.invoker`) can drive it deterministically. */
+export type InvokerStrategy = {
+    /** The source shape this strategy is for. */
+    readonly target: string;
+    /** Stable id for the invoker approach (used in README + report text). */
+    readonly kind: "single-turn-chat-loop" | "channel-resume-turn" | "voice-replay" | "gateway-request" | "chain-trigger" | "batch-item";
+    /** One-line human description of what the invoker wraps. */
+    readonly description: string;
+};
+export type ProjectEvalOptions = {
+    /**
+     * Dataset the eval bundle consumes. Defaults to `<specName>-eval` @ v1 on the
+     * dev split — the convention `dataset mine` / `distill --register` write to,
+     * so a shape's distilled feedback is discoverable without extra config.
+     */
+    readonly datasetName?: string;
+    readonly datasetVersion?: string;
+    readonly datasetSplit?: "train" | "dev" | "test";
+    /**
+     * Grader names for the projected bundle. Defaults to a single
+     * `substring_match` (a registry built-in) so the bundle compiles + runs
+     * credential-free; authors swap in their own graders.yaml-derived names.
+     */
+    readonly graders?: readonly {
+        readonly name: string;
+        readonly opts?: Record<string, unknown>;
+    }[];
+    readonly concurrency?: number;
+    readonly seed?: number;
+};
+/**
+ * Project a lowered IR node of ANY projectable shape into an `IrEvalV0`. The
+ * eval bundle emitted from the result reuses the source shape's agent verbatim,
+ * so the shape is evaluated on the exact model/instructions/tools it ships.
+ *
+ * Throws `EvalBridgeError` when the source is already `target: eval` (nothing to
+ * bridge) or is a multi-stage shape with no single agent.
+ */
+export declare function projectEvalIr(ir: IrNode, opts?: ProjectEvalOptions): ProjectedEvalIr;
+/**
+ * Select the per-shape AgentInvoker strategy an eval run should wrap the
+ * shape's runtime with. Exhaustive over the projectable shapes; unknown shapes
+ * fall back to the single-turn chat loop (the eval-runner default invoker).
+ */
+export declare function selectInvoker(target: string): InvokerStrategy;
+/** Where the eval bridge bundle is written, relative to the primary out-dir. */
+export declare const EVAL_BRIDGE_SUBDIR = "eval";
+/**
+ * Render the one-line summary the CLI prints after emitting an eval bridge, so
+ * the author knows which invoker seam to wrap for a faithful eval.
+ */
+export declare function describeBridge(projected: ProjectedEvalIr, strategy: InvokerStrategy): string;

package/dist/eval-bridge.js

@@ -0,0 +1,158 @@
+/** Thrown on an un-projectable shape / malformed bridge options. The CLI entry
+ *  file routes it through `die()`; tests assert on `.message`. */
+export class EvalBridgeError extends Error {
+    name = "EvalBridgeError";
+}
+/**
+ * Shapes that carry a single top-level `agent { model, instructions, tools }`
+ * and can therefore be projected 1:1 into an eval bundle. Multi-stage shapes
+ * (workflow/graph/crew/pipeline) are intentionally excluded — their eval
+ * bridge is a per-step projection, out of scope for slice 1.
+ */
+const PROJECTABLE_TARGETS = new Set([
+    "cli",
+    "channel",
+    "managed",
+    "voice",
+    "browser",
+    "onchain",
+    "onchain-game",
+    "batch",
+    "research",
+]);
+/** Multi-stage shapes with no single agent — a clearer error than "no model". */
+const MULTISTAGE_TARGETS = new Map([
+    ["workflow", "steps"],
+    ["graph", "nodes"],
+    ["crew", "roles"],
+    ["pipeline", "stages"],
+]);
+const DEFAULT_GRADERS = [{ name: "substring_match" }];
+const DEFAULT_CONCURRENCY = 2;
+/**
+ * Pull `{ model, instructions, tools }` off a lowered IR node, variant-
+ * agnostically. Returns undefined for shapes with no single agent sub-shape.
+ */
+function extractAgent(ir) {
+    const node = ir;
+    const agent = node.agent;
+    if (agent === undefined || typeof agent !== "object")
+        return undefined;
+    if (typeof agent.model !== "string" || typeof agent.instructions !== "string")
+        return undefined;
+    // Tools may live on `agent.tools` (eval/browser/channel/onchain/…) or on a
+    // sibling top-level `tools` (some variants). Prefer the agent-scoped list.
+    const tools = Array.isArray(agent.tools)
+        ? agent.tools
+        : Array.isArray(node.tools)
+            ? node.tools
+            : [];
+    return {
+        model: agent.model,
+        instructions: agent.instructions,
+        tools: tools.filter((t) => typeof t === "string"),
+    };
+}
+/**
+ * Project a lowered IR node of ANY projectable shape into an `IrEvalV0`. The
+ * eval bundle emitted from the result reuses the source shape's agent verbatim,
+ * so the shape is evaluated on the exact model/instructions/tools it ships.
+ *
+ * Throws `EvalBridgeError` when the source is already `target: eval` (nothing to
+ * bridge) or is a multi-stage shape with no single agent.
+ */
+export function projectEvalIr(ir, opts = {}) {
+    if (ir.target === "eval") {
+        throw new EvalBridgeError(`spec "${ir.name}" is already target: eval — no eval bridge needed (compile it directly)`);
+    }
+    if (ir.target === "cli") {
+        throw new EvalBridgeError(`spec "${ir.name}" is target: cli — use \`crewhaus eval\` directly (the eval bridge is for non-cli shapes)`);
+    }
+    const multistage = MULTISTAGE_TARGETS.get(ir.target);
+    if (multistage !== undefined) {
+        throw new EvalBridgeError(`spec "${ir.name}" is target: ${ir.target} — a multi-stage shape (${multistage}) has no single agent to project; per-step eval bridges are not yet supported`);
+    }
+    if (!PROJECTABLE_TARGETS.has(ir.target)) {
+        throw new EvalBridgeError(`spec "${ir.name}" target: ${ir.target} is not projectable into an eval bridge`);
+    }
+    const agent = extractAgent(ir);
+    if (agent === undefined) {
+        throw new EvalBridgeError(`spec "${ir.name}" (target: ${ir.target}) has no agent { model, instructions } to project into an eval bridge`);
+    }
+    const graders = opts.graders !== undefined && opts.graders.length > 0 ? opts.graders : DEFAULT_GRADERS;
+    const projected = {
+        version: 0,
+        name: ir.name,
+        target: "eval",
+        agent: { model: agent.model, instructions: agent.instructions, tools: agent.tools },
+        dataset: {
+            name: opts.datasetName ?? `${ir.name}-eval`,
+            version: opts.datasetVersion ?? "v1",
+            split: opts.datasetSplit ?? "dev",
+        },
+        graders: graders.map((g) => ({
+            name: g.name,
+            ...(g.opts !== undefined ? { opts: g.opts } : {}),
+        })),
+        concurrency: opts.concurrency ?? DEFAULT_CONCURRENCY,
+        ...(opts.seed !== undefined ? { seed: opts.seed } : {}),
+    };
+    return projected;
+}
+/**
+ * Select the per-shape AgentInvoker strategy an eval run should wrap the
+ * shape's runtime with. Exhaustive over the projectable shapes; unknown shapes
+ * fall back to the single-turn chat loop (the eval-runner default invoker).
+ */
+export function selectInvoker(target) {
+    switch (target) {
+        case "channel":
+            return {
+                target,
+                kind: "channel-resume-turn",
+                description: "wrap the channel-bot single-turn resume path (channel-adapter runTurn) so each dataset sample is a fresh inbound message",
+            };
+        case "voice":
+            return {
+                target,
+                kind: "voice-replay",
+                description: "replay recorded call-session transcripts through the voice runtime (see `crewhaus eval --voice`)",
+            };
+        case "managed":
+            return {
+                target,
+                kind: "gateway-request",
+                description: "drive the managed gateway's run handler per sample (one authenticated request per dataset row)",
+            };
+        case "onchain":
+        case "onchain-game":
+            return {
+                target,
+                kind: "chain-trigger",
+                description: "invoke the onchain daemon's trigger handler per sample against a simulated chain adapter (no live broadcast)",
+            };
+        case "batch":
+            return {
+                target,
+                kind: "batch-item",
+                description: "run one batch-worker item per dataset sample through the queue consumer",
+            };
+        default:
+            return {
+                target,
+                kind: "single-turn-chat-loop",
+                description: "single-turn runChatLoop over the shape's agent (the eval-runner default invoker)",
+            };
+    }
+}
+/** Where the eval bridge bundle is written, relative to the primary out-dir. */
+export const EVAL_BRIDGE_SUBDIR = "eval";
+/**
+ * Render the one-line summary the CLI prints after emitting an eval bridge, so
+ * the author knows which invoker seam to wrap for a faithful eval.
+ */
+export function describeBridge(projected, strategy) {
+    return (`eval bridge for target: ${strategy.target} → dataset ${projected.dataset.name}@${projected.dataset.version}` +
+        `#${projected.dataset.split}, graders [${projected.graders.map((g) => g.name).join(", ")}]; ` +
+        `invoker: ${strategy.kind} (${strategy.description})`);
+}