@skill-harness/core 0.17.0 → 0.18.0

package/dist/capture.js

@@ -1,344 +0,0 @@
-import { createHash } from "node:crypto";
-import { sep } from "node:path";
-import { CAPTURE_SCHEMA_VERSION } from "./capture-trace-types.js";
-/**
- * The chain of entries ending at `leafId`, in root→leaf order.
- *
- * With no `leafId`, the leaf is the last entry in file order — which is what
- * "the conversation I am currently in" means. Entries with no `id` (the
- * `session` header) are not part of the chain and are dropped.
- *
- * Cycles cannot occur in a well-formed file, but a corrupted one must not hang
- * the agent, so the walk is bounded by the entry count.
- */
-export function activeBranch(entries, leafId) {
-    const byId = new Map();
-    for (const e of entries)
-        if (typeof e.id === "string")
-            byId.set(e.id, e);
-    let cursor = leafId;
-    if (cursor === undefined) {
-        for (let i = entries.length - 1; i >= 0; i--) {
-            if (typeof entries[i].id === "string") {
-                cursor = entries[i].id;
-                break;
-            }
-        }
-    }
-    const chain = [];
-    const seen = new Set();
-    while (cursor !== undefined && cursor !== null && byId.has(cursor) && chain.length <= entries.length) {
-        if (seen.has(cursor))
-            break;
-        seen.add(cursor);
-        const entry = byId.get(cursor);
-        chain.push(entry);
-        cursor = entry.parentId ?? undefined;
-    }
-    return chain.reverse();
-}
-/** Extract visible text from content blocks. Thinking is dropped unconditionally. */
-export function visibleText(blocks) {
-    if (!blocks)
-        return "";
-    const parts = [];
-    for (const b of blocks) {
-        if (b.type === "thinking")
-            continue; // never, at any call site
-        if (b.type === "text" && typeof b.text === "string")
-            parts.push(b.text);
-        else if (b.type === "image")
-            parts.push("[image omitted]");
-    }
-    return parts.join("\n").trim();
-}
-/**
- * Group an already-branch-resolved entry list into logical turns.
- *
- * Entries that are not messages (`model_change`, `thinking_level_change`,
- * compaction markers, custom extension entries) are skipped rather than
- * guessed at: an unknown entry type is not evidence about the conversation.
- * Anything before the first user message is dropped — it belongs to no turn.
- */
-export function projectTurns(entries, homeDir) {
-    const turns = [];
-    let current = null;
-    for (const entry of entries) {
-        if (entry.type !== "message" || !entry.message)
-            continue;
-        const msg = entry.message;
-        const id = typeof entry.id === "string" ? entry.id : "";
-        if (msg.role === "user") {
-            current = {
-                index: turns.length,
-                user: visibleText(msg.content),
-                assistantText: "",
-                toolCalls: [],
-                entryIds: id ? [id] : [],
-            };
-            turns.push(current);
-            continue;
-        }
-        if (!current)
-            continue; // pre-conversation noise belongs to no turn
-        if (id)
-            current.entryIds.push(id);
-        if (msg.role === "assistant") {
-            const text = visibleText(msg.content);
-            if (text)
-                current.assistantText = current.assistantText ? `${current.assistantText}\n${text}` : text;
-            for (const b of msg.content ?? []) {
-                if (b.type !== "toolCall")
-                    continue;
-                current.toolCalls.push({
-                    name: typeof b.name === "string" ? b.name : "(unknown)",
-                    // Without `homeDir` this scrubbed secrets but left absolute home paths
-                    // intact — and these args are what the evidence sidecar records.
-                    args: redactArgs(b.arguments, homeDir),
-                    isError: false,
-                    ...(typeof b.id === "string" ? { id: b.id } : {}),
-                });
-            }
-            continue;
-        }
-        if (msg.role === "toolResult") {
-            // Attach the outcome to the matching call. Result BODIES are never kept —
-            // they routinely carry file contents, command output and absolute paths.
-            const body = JSON.stringify(msg.content ?? []);
-            const callId = typeof msg.toolCallId === "string" ? msg.toolCallId : undefined;
-            // By id when pi gave us one; otherwise the FIRST still-unmatched call of
-            // that name, since results arrive in order for a sequential conversation.
-            const target = callId
-                ? current.toolCalls.find((c) => c.id === callId)
-                : current.toolCalls.find((c) => c.name === msg.toolName && c.resultBytes === undefined);
-            if (target) {
-                target.isError = msg.isError === true;
-                target.resultBytes = Buffer.byteLength(body, "utf8");
-                target.resultSha256 = sha256(body);
-            }
-        }
-    }
-    return turns;
-}
-// ---------------------------------------------------------------------------
-// Redaction
-// ---------------------------------------------------------------------------
-/** Values longer than this are truncated rather than persisted whole. */
-export const MAX_VALUE_CHARS = 2000;
-const REDACTED = "[redacted]";
-/** Argument/field names whose VALUE is always dropped, whatever it looks like. */
-const SECRET_KEY = /^(.*[-_])?(password|passwd|secret|token|api[-_]?key|apikey|auth|authorization|credential|private[-_]?key|access[-_]?key|session[-_]?key)([-_].*)?$/i;
-/** Value-shaped secrets, caught even under an innocuous key name. */
-const SECRET_VALUE = [
-    /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,
-    /\bBearer\s+[A-Za-z0-9._~+/-]{16,}=*/g,
-    /\bsk-[A-Za-z0-9]{16,}\b/g,
-    /\bgh[pousr]_[A-Za-z0-9]{16,}\b/g,
-    /\bxox[abposr]-[A-Za-z0-9-]{10,}\b/g,
-    /\bAKIA[0-9A-Z]{16}\b/g,
-    /\bey[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g, // JWT
-    // Credentials embedded in a URL — `postgres://user:pass@host/db`. Caught by
-    // shape rather than by key name: the key is usually something like `DB_URL`,
-    // which no list of secret-sounding names will ever match.
-    /\b([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s@]+@/gi,
-];
-/**
- * Redact secrets and machine paths from a free-text string.
- *
- * Home directories are replaced because a capture is meant to be committed, and
- * `/home/<name>/…` identifies a person as surely as a token identifies an
- * account. `homeDir` is a parameter rather than a read of `process.env.HOME` so
- * the behavior is testable and does not differ between a developer's machine
- * and CI.
- */
-export function redactText(input, homeDir) {
-    let out = input;
-    // The URL pattern keeps its scheme so the value stays recognisable as a URL;
-    // every other pattern replaces the whole match.
-    out = out.replace(SECRET_VALUE[SECRET_VALUE.length - 1], `$1${REDACTED}@`);
-    for (const re of SECRET_VALUE.slice(0, -1))
-        out = out.replace(re, REDACTED);
-    if (homeDir && homeDir.length > 1) {
-        out = out.split(homeDir).join("~");
-    }
-    return out;
-}
-/** Truncate with an explicit marker — silent truncation reads as complete evidence. */
-export function truncate(input, max = MAX_VALUE_CHARS) {
-    if (input.length <= max)
-        return input;
-    return `${input.slice(0, max)}… [truncated ${input.length - max} chars]`;
-}
-/**
- * Sanitize tool-call arguments: drop secret-named values, redact secret-shaped
- * ones, truncate the oversized, and refuse to recurse without bound.
- */
-export function redactArgs(args, homeDir, depth = 0) {
-    if (args === null || typeof args !== "object" || Array.isArray(args))
-        return {};
-    const out = {};
-    for (const [key, value] of Object.entries(args)) {
-        if (SECRET_KEY.test(key)) {
-            out[key] = REDACTED;
-            continue;
-        }
-        out[key] = redactValue(value, homeDir, depth);
-    }
-    return out;
-}
-function redactValue(value, homeDir, depth) {
-    if (typeof value === "string")
-        return truncate(redactText(value, homeDir));
-    if (typeof value === "number" || typeof value === "boolean" || value === null)
-        return value;
-    if (depth >= 3)
-        return "[nested]"; // bounded: a deep object is not review evidence
-    if (Array.isArray(value))
-        return value.slice(0, 20).map((v) => redactValue(v, homeDir, depth + 1));
-    if (typeof value === "object")
-        return redactArgs(value, homeDir, depth + 1);
-    return String(value);
-}
-function sha256(text) {
-    return createHash("sha256").update(text, "utf8").digest("hex");
-}
-// ---------------------------------------------------------------------------
-// Building a capture case
-// ---------------------------------------------------------------------------
-/** Stable, sortable, collision-resistant enough for a per-skill directory. */
-export function captureId(seed, existing = []) {
-    const taken = new Set(existing);
-    const base = `CAP-${sha256(seed).slice(0, 6).toUpperCase()}`;
-    if (!taken.has(base))
-        return base;
-    for (let n = 2;; n++) {
-        const candidate = `${base}-${n}`;
-        if (!taken.has(candidate))
-            return candidate;
-    }
-}
-/**
- * Assemble a reviewed capture case.
- *
- * Only the USER turns are carried into the case: they are the stimulus, and they
- * are the only part that can be replayed. The assistant's historical prose is
- * evidence for the human writing the expectation, never an exact-output oracle —
- * so it lives in the git-ignored local sidecar, not here.
- *
- * The session path is hashed rather than stored: an absolute path names a
- * machine and a user, and a hash is enough to recognize the same session again.
- */
-export function buildCaptureCase(opts) {
-    const { start, end } = opts.range;
-    if (!Number.isInteger(start) || !Number.isInteger(end) || start < 0 || end < start || end >= opts.turns.length) {
-        throw new Error(`invalid capture range ${start}..${end} over ${opts.turns.length} turn(s)`);
-    }
-    if (opts.expectedBehavior.trim() === "") {
-        throw new Error("a capture needs a written expected behavior — it is what makes the case reviewable");
-    }
-    const checklist = opts.checklist.map((c) => c.trim()).filter(Boolean);
-    if (checklist.length === 0) {
-        throw new Error("a capture needs at least one checklist item");
-    }
-    const selected = opts.turns.slice(start, end + 1);
-    const turns = selected.map((t) => truncate(redactText(t.user, opts.homeDir)));
-    const id = captureId(`${opts.sessionPath}:${start}:${end}:${opts.created}`, opts.existingIds ?? []);
-    return {
-        capture_schema: CAPTURE_SCHEMA_VERSION,
-        id,
-        created: opts.created,
-        classification: opts.classification,
-        turns,
-        expected_behavior: truncate(redactText(opts.expectedBehavior, opts.homeDir)),
-        checklist: checklist.map((c) => truncate(redactText(c, opts.homeDir), 300)),
-        target: opts.target,
-        // `covers` refs resolve against the SPEC dir, and `target.path` is relative
-        // to the skill root — one level up. A subagent path (`.pi/agents/x.md`) is
-        // carried the same way; both are instruction files a section walk can read.
-        covers: [`../${opts.target.path.split(sep).join("/")}`],
-        provenance: {
-            session_sha256: sha256(opts.sessionPath),
-            turn_range: { start, end },
-            ...(opts.subject ? { subject: opts.subject } : {}),
-            ...(opts.gitCommit ? { git_commit: opts.gitCommit } : {}),
-            ...(opts.gitDirty === undefined ? {} : { git_dirty: opts.gitDirty }),
-        },
-        status: "pending",
-    };
-}
-/**
- * Project a reviewed capture into a scenario object for `appendScenario`.
- *
- * Deliberately minimal: id, title, turns, checklist. No `critical`, no gates, no
- * fixture. Promotion is the moment a human takes responsibility for a test, and
- * a capture cannot know whether the behavior it saw is ship-blocking — guessing
- * `critical: true` here would let a captured one-off silently gate a release.
- */
-export function captureToScenario(capture, scenarioId, title) {
-    return {
-        id: scenarioId,
-        title,
-        turns: capture.turns,
-        checklist: capture.checklist,
-    };
-}
-/**
- * Draft an orchestration assertion from subagent calls seen in the captured range.
- *
- * Returns null when the range contains none, so the capture UI can skip the
- * question entirely rather than offering an empty form.
- *
- * Deliberately proposes only `agent` and `count` — never `task_contains`. The
- * task text that happened to be sent is not the task text that is *required*;
- * turning one observed handoff into a required substring would manufacture a
- * brittle assertion the author never reasoned about. Required context is a
- * judgement, so the UI asks.
- */
-export function draftSubagentAssertion(turns, toolNames = ["Agent", "subagent", "task"]) {
-    const names = new Set(toolNames.map((n) => n.toLowerCase()));
-    for (const turn of turns) {
-        for (const call of turn.toolCalls) {
-            if (!names.has(call.name.toLowerCase()))
-                continue;
-            const invocations = subagentInvocationsOf(call.args);
-            if (invocations.length === 0)
-                continue;
-            return { tool: call.name, agent: invocations[0], count: { min: 1 } };
-        }
-    }
-    return null;
-}
-/** Agent names visible in a call's arguments, across the known shapes. */
-function subagentInvocationsOf(args) {
-    const nameOf = (v) => {
-        if (v === null || typeof v !== "object" || Array.isArray(v))
-            return null;
-        const o = v;
-        if (typeof o.agent === "string")
-            return o.agent;
-        if (typeof o.name === "string")
-            return o.name;
-        return null;
-    };
-    for (const key of ["tasks", "chain"]) {
-        const list = args[key];
-        if (Array.isArray(list))
-            return list.map(nameOf).filter((x) => x !== null);
-    }
-    const single = nameOf(args);
-    return single ? [single] : [];
-}
-/**
- * Offline first-draft checklist from the human's expectation.
- *
- * Splits on sentence and bullet boundaries. This is a typing shortcut for the
- * editor step, not an understanding of the text — the UI always opens the result
- * for correction, and the plan makes LLM drafting a separate, paid, opt-in path.
- */
-export function draftChecklist(expectedBehavior) {
-    return expectedBehavior
-        .split(/\n\s*[-*]\s+|\n{2,}|(?<=[.!?])\s+(?=[A-Z])/)
-        .map((s) => s.replace(/^[-*]\s+/, "").replace(/\s+/g, " ").trim().replace(/[.]$/, ""))
-        .filter((s) => s.length > 3);
-}
-//# sourceMappingURL=capture.js.map

package/dist/comparison.d.ts

@@ -1,97 +0,0 @@
-import type { ResultsFile, ScenarioResult } from "./results.js";
-import type { Verdict } from "./score.js";
-export interface ComparisonSideDigests {
-    skill: string;
-    spec: string;
-    fixtures: Record<string, string>;
-    /** Every non-candidate test input: scenario facets, persona, fixtures, extensions, post-tests, and system prompts. */
-    inputs: Record<string, string>;
-}
-export interface ComparisonDigests {
-    reference: ComparisonSideDigests;
-    candidate: ComparisonSideDigests;
-    harness: string;
-    model: string;
-    judge: string;
-    environment: {
-        node: string;
-        platform: string;
-        arch: string;
-        [key: string]: string;
-    };
-}
-export interface ComparisonThresholds {
-    max_subject_token_increase?: number;
-    max_wall_time_increase?: number;
-    max_tool_call_increase?: number;
-}
-export interface ComparisonCell {
-    id: string;
-    reference: Verdict;
-    candidate: Verdict;
-    change: "regression" | "lift" | "held-pass" | "held-fail" | "inconclusive";
-    critical: boolean;
-    reference_flakiness: number | null;
-    candidate_flakiness: number | null;
-}
-export interface AggregateMetrics {
-    wall_time_ms: number;
-    judge_calls: number;
-    judge_rejudge_calls: number;
-    subject_metrics_reps: number;
-    total_reps: number;
-    input_tokens: number | null;
-    output_tokens: number | null;
-    cache_read_tokens: number | null;
-    cache_write_tokens: number | null;
-    subject_cost_usd: number | null;
-    cost_source: "provider-reported" | "subscription" | "unreported" | null;
-    tool_calls: number | null;
-    delegated_children: number | null;
-    max_concurrency: number | null;
-}
-export interface ComparisonReport {
-    schema: 1;
-    skill: string;
-    model: string;
-    mode: string;
-    reps: number;
-    judge: string;
-    paired_setup: true;
-    deterministic_sampling: false;
-    sampling_note: string;
-    partial: boolean;
-    release_eligible: boolean;
-    ship: boolean;
-    note: string;
-    cells: ComparisonCell[];
-    behavioral_regressions: string[];
-    critical_regressions: string[];
-    infrastructure_errors: string[];
-    metrics: {
-        reference: AggregateMetrics;
-        candidate: AggregateMetrics;
-    };
-    cost_regressions: string[];
-    thresholds?: ComparisonThresholds;
-    digests: ComparisonDigests;
-    reference: ResultsFile;
-    candidate: ResultsFile;
-}
-export declare function buildComparison(input: {
-    skill: string;
-    model: string;
-    mode: string;
-    reps: number;
-    judge: string;
-    reference: ResultsFile;
-    candidate: ResultsFile;
-    critical: string[];
-    digests: ComparisonDigests;
-    partial?: boolean;
-    thresholds?: ComparisonThresholds;
-}): ComparisonReport;
-/** 2 = critical behavioral regression, 1 = ordinary/ship/infra regression, 0 = clean or partial feedback. */
-export declare function comparisonExitCode(report: ComparisonReport): 0 | 1 | 2;
-export declare function formatComparison(report: ComparisonReport): string;
-export declare function aggregateMetrics(scenarios: ScenarioResult[]): AggregateMetrics;

package/dist/comparison.js

@@ -1,189 +0,0 @@
-export function buildComparison(input) {
-    const { reference, candidate } = input;
-    if (reference.model !== candidate.model || reference.model !== input.model)
-        throw new Error("comparison subject models differ");
-    if (reference.mode !== candidate.mode || reference.mode !== input.mode)
-        throw new Error("comparison delivery modes differ");
-    const referenceJudge = `${reference.judge.provider}:${reference.judge.model}`;
-    const candidateJudge = `${candidate.judge.provider}:${candidate.judge.model}`;
-    if (referenceJudge !== candidateJudge || referenceJudge !== input.judge)
-        throw new Error("comparison judge policies differ");
-    const referenceIds = reference.scenarios.map((scenario) => scenario.id);
-    const candidateIds = candidate.scenarios.map((scenario) => scenario.id);
-    if (JSON.stringify([...referenceIds].sort()) !== JSON.stringify([...candidateIds].sort())) {
-        throw new Error("comparison scenario IDs differ — paired comparisons require the same scenario plan");
-    }
-    if (input.digests.reference.spec !== input.digests.candidate.spec ||
-        JSON.stringify(sorted(input.digests.reference.fixtures)) !== JSON.stringify(sorted(input.digests.candidate.fixtures)) ||
-        JSON.stringify(sorted(input.digests.reference.inputs)) !== JSON.stringify(sorted(input.digests.candidate.inputs))) {
-        throw new Error("comparison spec/test-input digests differ — only the skill candidate may change in a paired run");
-    }
-    const referencePlan = Object.fromEntries(reference.scenarios.map((scenario) => [scenario.id, [scenario.reps ?? 1, scenario.pass_threshold ?? null]]));
-    const candidatePlan = Object.fromEntries(candidate.scenarios.map((scenario) => [scenario.id, [scenario.reps ?? 1, scenario.pass_threshold ?? null]]));
-    if (JSON.stringify(sorted(referencePlan)) !== JSON.stringify(sorted(candidatePlan))) {
-        throw new Error("comparison repetition/threshold plans differ");
-    }
-    const critical = new Set(input.critical);
-    const byCandidate = new Map(candidate.scenarios.map((scenario) => [scenario.id, scenario]));
-    const cells = reference.scenarios.map((left) => {
-        const right = byCandidate.get(left.id);
-        const a = effective(left), b = effective(right);
-        const conclusive = !untrustworthy(left, a) && !untrustworthy(right, b);
-        const change = !conclusive
-            ? "inconclusive"
-            : a === "PASS" && b !== "PASS"
-                ? "regression"
-                : a !== "PASS" && b === "PASS"
-                    ? "lift"
-                    : a === "PASS"
-                        ? "held-pass"
-                        : "held-fail";
-        return {
-            id: left.id,
-            reference: a,
-            candidate: b,
-            change,
-            critical: critical.has(left.id),
-            reference_flakiness: left.flakiness ?? null,
-            candidate_flakiness: right.flakiness ?? null,
-        };
-    });
-    const behavioral = cells.filter((cell) => cell.change === "regression").map((cell) => cell.id);
-    const criticalRegressions = cells.filter((cell) => cell.change === "regression" && cell.critical).map((cell) => cell.id);
-    const byReference = new Map(reference.scenarios.map((scenario) => [scenario.id, scenario]));
-    const infrastructureErrors = cells.filter((cell) => cell.reference === "ERROR" || cell.reference === "NOT-MEASURED" || cell.reference === "JUDGE-AMBIGUOUS" || byReference.get(cell.id)?.suspect ||
-        cell.candidate === "ERROR" || cell.candidate === "NOT-MEASURED" || cell.candidate === "JUDGE-AMBIGUOUS" || byCandidate.get(cell.id)?.suspect).map((cell) => cell.id);
-    const metrics = { reference: aggregateMetrics(reference.scenarios), candidate: aggregateMetrics(candidate.scenarios) };
-    const costRegressions = thresholdFindings(metrics.reference, metrics.candidate, input.thresholds);
-    const partial = input.partial === true || reference.partial === true || candidate.partial === true;
-    const releaseEligible = !partial;
-    const ship = releaseEligible && candidate.effective_grade.ship && behavioral.length === 0 && infrastructureErrors.length === 0 && costRegressions.length === 0;
-    const note = partial
-        ? "branch feedback only — partial/affected comparisons never SHIP; run the full paired plan for a release decision"
-        : ship
-            ? "candidate clears its ship bar with no paired behavioral regression"
-            : "candidate is NOT READY under the paired behavioral/cost release policy";
-    return {
-        schema: 1,
-        skill: input.skill,
-        model: input.model,
-        mode: input.mode,
-        reps: input.reps,
-        judge: input.judge,
-        paired_setup: true,
-        deterministic_sampling: false,
-        sampling_note: "paired setup only: prompts/fixtures/models/policy/repetition plan match; provider-deterministic seeding is not claimed",
-        partial,
-        release_eligible: releaseEligible,
-        ship,
-        note,
-        cells,
-        behavioral_regressions: behavioral,
-        critical_regressions: criticalRegressions,
-        infrastructure_errors: [...new Set(infrastructureErrors)],
-        metrics,
-        cost_regressions: costRegressions,
-        ...(input.thresholds ? { thresholds: input.thresholds } : {}),
-        digests: input.digests,
-        reference,
-        candidate,
-    };
-}
-/** 2 = critical behavioral regression, 1 = ordinary/ship/infra regression, 0 = clean or partial feedback. */
-export function comparisonExitCode(report) {
-    if (report.critical_regressions.length)
-        return 2;
-    if (report.behavioral_regressions.length || report.infrastructure_errors.length || report.cost_regressions.length || (report.release_eligible && !report.ship))
-        return 1;
-    return 0;
-}
-export function formatComparison(report) {
-    const lines = [
-        `── compare ${report.skill} · ${report.model} · mode=${report.mode} ×${report.reps} ──`,
-        "  sampling: paired setup; provider-deterministic seeding NOT claimed",
-    ];
-    for (const cell of report.cells) {
-        const mark = cell.change === "regression" ? "↓" : cell.change === "lift" ? "↑" : cell.change === "inconclusive" ? "?" : "=";
-        const variance = cell.reference_flakiness === null && cell.candidate_flakiness === null
-            ? ""
-            : ` · flakiness ${cell.reference_flakiness ?? "n/a"} → ${cell.candidate_flakiness ?? "n/a"}`;
-        lines.push(`  ${mark} ${cell.id}${cell.critical ? " CRITICAL" : ""}: ${cell.reference} → ${cell.candidate} (${cell.change})${variance}`);
-    }
-    lines.push(`  behavior: ${report.behavioral_regressions.length} regression(s), ${report.cells.filter((cell) => cell.change === "lift").length} lift(s), ${report.infrastructure_errors.length} infrastructure/inconclusive error(s)`);
-    lines.push(`  subject tokens: input ${metricDelta(report.metrics.reference.input_tokens, report.metrics.candidate.input_tokens)} · output ${metricDelta(report.metrics.reference.output_tokens, report.metrics.candidate.output_tokens)} · cache-read ${metricDelta(report.metrics.reference.cache_read_tokens, report.metrics.candidate.cache_read_tokens)} (${report.metrics.candidate.subject_metrics_reps}/${report.metrics.candidate.total_reps} candidate reps reported)`);
-    lines.push(`  judge calls: ${report.metrics.reference.judge_calls} → ${report.metrics.candidate.judge_calls} (re-judge ${report.metrics.reference.judge_rejudge_calls} → ${report.metrics.candidate.judge_rejudge_calls})`);
-    lines.push(`  wall time: ${report.metrics.reference.wall_time_ms}ms → ${report.metrics.candidate.wall_time_ms}ms`);
-    lines.push(`  tool calls: ${metricDelta(report.metrics.reference.tool_calls, report.metrics.candidate.tool_calls)}`);
-    if (report.cost_regressions.length)
-        lines.push(`  cost/latency thresholds (separate from behavior): ${report.cost_regressions.join("; ")}`);
-    lines.push(`  ${report.ship ? "SHIP" : "NOT READY"}: ${report.note}`);
-    return lines.join("\n");
-}
-export function aggregateMetrics(scenarios) {
-    const metrics = scenarios.map((scenario) => scenario.metrics).filter((value) => value !== undefined);
-    const sumOptional = (field) => {
-        const values = metrics.map((value) => value[field]).filter((value) => typeof value === "number");
-        return values.length ? values.reduce((sum, value) => sum + value, 0) : null;
-    };
-    const maxValues = metrics.map((value) => value.max_concurrency).filter((value) => typeof value === "number");
-    return {
-        wall_time_ms: metrics.reduce((sum, value) => sum + value.wall_time_ms, 0),
-        judge_calls: metrics.reduce((sum, value) => sum + value.judge_calls, 0),
-        judge_rejudge_calls: metrics.reduce((sum, value) => sum + value.judge_rejudge_calls, 0),
-        subject_metrics_reps: metrics.reduce((sum, value) => sum + value.subject_metrics_reps, 0),
-        total_reps: metrics.reduce((sum, value) => sum + value.total_reps, 0),
-        input_tokens: sumOptional("input_tokens"),
-        output_tokens: sumOptional("output_tokens"),
-        cache_read_tokens: sumOptional("cache_read_tokens"),
-        cache_write_tokens: sumOptional("cache_write_tokens"),
-        subject_cost_usd: sumOptional("subject_cost_usd"),
-        cost_source: (() => {
-            const sources = scenarios.map((scenario) => scenario.metrics?.cost_source).filter((source) => source !== undefined);
-            return sources.length === 0 ? null : sources.every((source) => source === sources[0]) ? sources[0] : "unreported";
-        })(),
-        tool_calls: sumOptional("tool_calls"),
-        delegated_children: sumOptional("delegated_children"),
-        max_concurrency: maxValues.length ? Math.max(...maxValues) : null,
-    };
-}
-function effective(result) {
-    if (result.override)
-        return result.override;
-    if (result.objective?.status === "ERROR")
-        return "ERROR";
-    if (result.objective?.status === "NOT-MEASURED")
-        return "NOT-MEASURED";
-    if (result.objective?.status === "FAIL")
-        return "FAIL";
-    return result.judge_verdict;
-}
-function untrustworthy(result, verdict) { return result.suspect || verdict === "ERROR" || verdict === "NOT-MEASURED" || verdict === "JUDGE-AMBIGUOUS"; }
-function sorted(value) { return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))); }
-function increase(reference, candidate) { return reference === null || candidate === null || reference <= 0 ? null : (candidate - reference) / reference; }
-function thresholdFindings(reference, candidate, thresholds) {
-    if (!thresholds)
-        return [];
-    const out = [];
-    const check = (label, left, right, max) => {
-        if (max === undefined)
-            return;
-        if (left === null || right === null) {
-            out.push(`${label} unavailable/incomplete — cannot evaluate configured ${(max * 100).toFixed(1)}% maximum`);
-            return;
-        }
-        if (left === 0) {
-            if (right > 0)
-                out.push(`${label} increased from 0 to ${right}, exceeding the configured maximum`);
-            return;
-        }
-        const delta = increase(left, right);
-        if (delta > max)
-            out.push(`${label} +${(delta * 100).toFixed(1)}% exceeds ${(max * 100).toFixed(1)}%`);
-    };
-    check("subject tokens", reference.input_tokens, candidate.input_tokens, thresholds.max_subject_token_increase);
-    check("wall time", reference.wall_time_ms, candidate.wall_time_ms, thresholds.max_wall_time_increase);
-    check("tool calls", reference.tool_calls, candidate.tool_calls, thresholds.max_tool_call_increase);
-    return out;
-}
-function metricDelta(reference, candidate) { return reference === null || candidate === null ? "unavailable" : `${reference} → ${candidate}`; }
-//# sourceMappingURL=comparison.js.map