@tangle-network/agent-eval 0.93.0 → 0.95.0

package/dist/reference-replay.d.ts

@@ -0,0 +1,176 @@
+/**
+ * Reference replay — score an agent against withheld historical outcomes.
+ *
+ * This is the generic version of the public-audit replay pattern:
+ * run a candidate system on an old task, keep the reference answers hidden
+ * until after execution, then score recall/precision and gate promotion
+ * across train/dev/test/holdout splits.
+ */
+export type ReferenceReplaySplit = 'train' | 'dev' | 'test' | 'holdout';
+export type ReferenceReplayMatchStrategy = 'reference-order' | 'global-greedy';
+export interface ReferenceReplayItem {
+    id: string;
+    title: string;
+    description?: string;
+    severity?: string;
+    tags?: string[];
+    weight?: number;
+}
+export interface ReferenceReplayCandidate {
+    id: string;
+    title: string;
+    description?: string;
+    severity?: string;
+    tags?: string[];
+    metadata?: Record<string, unknown>;
+}
+export interface ReferenceReplayScenario {
+    id: string;
+    split?: ReferenceReplaySplit;
+    references: ReferenceReplayItem[];
+    candidates: ReferenceReplayCandidate[];
+    metadata?: Record<string, unknown>;
+}
+export interface ReferenceReplayCase<Input = unknown> {
+    id: string;
+    split?: ReferenceReplaySplit;
+    input: Input;
+    references: ReferenceReplayItem[];
+    metadata?: Record<string, unknown>;
+}
+export interface ReferenceReplayExecutionScenario<Input = unknown> {
+    id: string;
+    split: ReferenceReplaySplit;
+    input: Input;
+    metadata?: Record<string, unknown>;
+}
+export interface ReferenceReplayRunContext {
+    runId: string;
+    caseIndex: number;
+    abortSignal?: AbortSignal;
+}
+export interface ReferenceReplayAdapter<Input = unknown> {
+    run(scenario: ReferenceReplayExecutionScenario<Input>, context: ReferenceReplayRunContext): Promise<ReferenceReplayCandidate[]>;
+}
+export type ReferenceReplayAdapterFn<Input = unknown> = (scenario: ReferenceReplayExecutionScenario<Input>, context: ReferenceReplayRunContext) => Promise<ReferenceReplayCandidate[]>;
+export type ReferenceReplayAdapterLike<Input = unknown> = ReferenceReplayAdapter<Input> | ReferenceReplayAdapterFn<Input>;
+export interface ReferenceReplayMatch {
+    scenarioId: string;
+    referenceId: string;
+    candidateId: string | null;
+    score: number;
+    matched: boolean;
+    weight: number;
+    reason: string;
+}
+export interface ReferenceReplayScenarioScore {
+    scenarioId: string;
+    split: ReferenceReplaySplit;
+    matched: number;
+    total: number;
+    falsePositives: number;
+    matchedWeight: number;
+    totalWeight: number;
+    precision: number;
+    recall: number;
+    f1: number;
+    matches: ReferenceReplayMatch[];
+}
+export interface ReferenceReplayAggregate {
+    matched: number;
+    total: number;
+    falsePositives: number;
+    matchedWeight: number;
+    totalWeight: number;
+    precision: number;
+    recall: number;
+    f1: number;
+    weightedRecall: number;
+}
+export interface ReferenceReplayScore {
+    scenarios: ReferenceReplayScenarioScore[];
+    aggregate: ReferenceReplayAggregate;
+    bySplit: Partial<Record<ReferenceReplaySplit, ReferenceReplayAggregate>>;
+}
+export interface ReferenceMatchResult {
+    score: number;
+    reason?: string;
+}
+export type ReferenceReplayMatcher = (reference: ReferenceReplayItem, candidate: ReferenceReplayCandidate, scenario: ReferenceReplayScenario) => ReferenceMatchResult;
+export interface ReferenceReplayScoreOptions {
+    matcher?: ReferenceReplayMatcher;
+    matchThreshold?: number;
+    matchStrategy?: ReferenceReplayMatchStrategy;
+    includeHoldout?: boolean;
+    splits?: ReferenceReplaySplit[];
+}
+export interface ReferenceReplayPromotionPolicy {
+    /** Splits that must improve or stay flat. Default: ['dev', 'test']. */
+    requiredSplits?: ReferenceReplaySplit[];
+    /** Minimum aggregate F1 lift required on required splits. Default 0. */
+    minF1Delta?: number;
+    /** Maximum F1 drop allowed on any compared split. Default 0. */
+    maxRegression?: number;
+    /** If true, holdout must be present and must not regress. Default true. */
+    requireHoldoutNonRegression?: boolean;
+}
+export interface ReferenceReplaySplitComparison {
+    split: ReferenceReplaySplit;
+    baselineF1: number;
+    candidateF1: number;
+    f1Delta: number;
+    baselineRecall: number;
+    candidateRecall: number;
+    recallDelta: number;
+}
+export interface ReferenceReplayPromotionDecision {
+    promote: boolean;
+    reason: string;
+    aggregateDelta: number;
+    comparisons: ReferenceReplaySplitComparison[];
+    regressions: ReferenceReplaySplitComparison[];
+}
+export interface ReferenceReplayCaseRun<Input = unknown> {
+    caseId: string;
+    split: ReferenceReplaySplit;
+    input: Input;
+    metadata?: Record<string, unknown>;
+    references: ReferenceReplayItem[];
+    candidates: ReferenceReplayCandidate[];
+    score: ReferenceReplayScenarioScore;
+    durationMs: number;
+    error?: string;
+}
+export interface ReferenceReplayRun<Input = unknown> {
+    id: string;
+    variantId?: string;
+    startedAt: number;
+    completedAt: number;
+    durationMs: number;
+    cases: ReferenceReplayCaseRun<Input>[];
+    score: ReferenceReplayScore;
+    metadata?: Record<string, unknown>;
+}
+export interface ReferenceReplayRunOptions<Input = unknown> extends ReferenceReplayScoreOptions {
+    adapter: ReferenceReplayAdapterLike<Input>;
+    runId?: string;
+    variantId?: string;
+    metadata?: Record<string, unknown>;
+    store?: ReferenceReplayRunStore<Input>;
+    abortSignal?: AbortSignal;
+    continueOnError?: boolean;
+    now?: () => number;
+}
+export interface ReferenceReplayRunStore<Input = unknown> {
+    save(run: ReferenceReplayRun<Input>): Promise<void>;
+    list(): Promise<ReferenceReplayRun<Input>[]>;
+}
+export declare function runReferenceReplay<Input = unknown>(cases: ReferenceReplayCase<Input>[], options: ReferenceReplayRunOptions<Input>): Promise<ReferenceReplayRun<Input>>;
+export declare function decideReferenceReplayRunPromotion(baseline: ReferenceReplayRun, candidate: ReferenceReplayRun, policy?: ReferenceReplayPromotionPolicy): ReferenceReplayPromotionDecision;
+export declare function inMemoryReferenceReplayStore<Input = unknown>(initial?: ReferenceReplayRun<Input>[]): ReferenceReplayRunStore<Input>;
+export declare function jsonlReferenceReplayStore<Input = unknown>(path: string): ReferenceReplayRunStore<Input>;
+export declare function scoreReferenceReplay(scenarios: ReferenceReplayScenario[], options?: ReferenceReplayScoreOptions): ReferenceReplayScore;
+export declare function compareReferenceReplay(baseline: ReferenceReplayScore, candidate: ReferenceReplayScore): ReferenceReplaySplitComparison[];
+export declare function decideReferenceReplayPromotion(baseline: ReferenceReplayScore, candidate: ReferenceReplayScore, policy?: ReferenceReplayPromotionPolicy): ReferenceReplayPromotionDecision;
+export declare function defaultReferenceReplayMatcher(reference: ReferenceReplayItem, candidate: ReferenceReplayCandidate): ReferenceMatchResult;
+//# sourceMappingURL=reference-replay.d.ts.map

package/dist/reference-replay.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reference-replay.d.ts","sourceRoot":"","sources":["../src/reference-replay.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,MAAM,MAAM,oBAAoB,GAAG,OAAO,GAAG,KAAK,GAAG,MAAM,GAAG,SAAS,CAAA;AACvE,MAAM,MAAM,4BAA4B,GAAG,iBAAiB,GAAG,eAAe,CAAA;AAE9E,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,wBAAwB;IACvC,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,CAAC,EAAE,oBAAoB,CAAA;IAC5B,UAAU,EAAE,mBAAmB,EAAE,CAAA;IACjC,UAAU,EAAE,wBAAwB,EAAE,CAAA;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,mBAAmB,CAAC,KAAK,GAAG,OAAO;IAClD,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,CAAC,EAAE,oBAAoB,CAAA;IAC5B,KAAK,EAAE,KAAK,CAAA;IACZ,UAAU,EAAE,mBAAmB,EAAE,CAAA;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,gCAAgC,CAAC,KAAK,GAAG,OAAO;IAC/D,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,oBAAoB,CAAA;IAC3B,KAAK,EAAE,KAAK,CAAA;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,CAAC,EAAE,WAAW,CAAA;CAC1B;AAED,MAAM,WAAW,sBAAsB,CAAC,KAAK,GAAG,OAAO;IACrD,GAAG,CACD,QAAQ,EAAE,gCAAgC,CAAC,KAAK,CAAC,EACjD,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,wBAAwB,EAAE,CAAC,CAAA;CACvC;AAED,MAAM,MAAM,wBAAwB,CAAC,KAAK,GAAG,OAAO,IAAI,CACtD,QAAQ,EAAE,gCAAgC,CAAC,KAAK,CAAC,EACjD,OAAO,EAAE,yBAAyB,KAC/B,OAAO,CAAC,wBAAwB,EAAE,CAAC,CAAA;AAExC,MAAM,MAAM,0BAA0B,CAAC,KAAK,GAAG,OAAO,IAClD,sBAAsB,CAAC,KAAK,CAAC,GAC7B,wBAAwB,CAAC,KAAK,CAAC,CAAA;AAEnC,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;IAC1B,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,OAAO,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,4BAA4B;IAC3C,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,oBAAoB,CAAA;IAC3B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IACrB,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,oBAAoB,EAAE,CAAA;CAChC;AAED,MAAM,WAAW,wBAAwB;IACvC,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IACrB,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,EAAE,EAAE,MAAM,CAAA;IACV,cAAc,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,4BAA4B,EAAE,CAAA;IACzC,SAAS,EAAE,wBAAwB,CAAA;IACnC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,oBAAoB,EAAE,wBAAwB,CAAC,CAAC,CAAA;CACzE;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,MAAM,sBAAsB,GAAG,CACnC,SAAS,EAAE,mBAAmB,EAC9B,SAAS,EAAE,wBAAwB,EACnC,QAAQ,EAAE,uBAAuB,KAC9B,oBAAoB,CAAA;AAEzB,MAAM,WAAW,2BAA2B;IAC1C,OAAO,CAAC,EAAE,sBAAsB,CAAA;IAChC,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,aAAa,CAAC,EAAE,4BAA4B,CAAA;IAC5C,cAAc,CAAC,EAAE,OAAO,CAAA;IACxB,MAAM,CAAC,EAAE,oBAAoB,EAAE,CAAA;CAChC;AAED,MAAM,WAAW,8BAA8B;IAC7C,uEAAuE;IACvE,cAAc,CAAC,EAAE,oBAAoB,EAAE,CAAA;IACvC,wEAAwE;IACxE,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,gEAAgE;IAChE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,2EAA2E;IAC3E,2BAA2B,CAAC,EAAE,OAAO,CAAA;CACtC;AAED,MAAM,WAAW,8BAA8B;IAC7C,KAAK,EAAE,oBAAoB,CAAA;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,cAAc,EAAE,MAAM,CAAA;IACtB,eAAe,EAAE,MAAM,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,gCAAgC;IAC/C,OAAO,EAAE,OAAO,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,cAAc,EAAE,MAAM,CAAA;IACtB,WAAW,EAAE,8BAA8B,EAAE,CAAA;IAC7C,WAAW,EAAE,8BAA8B,EAAE,CAAA;CAC9C;AAED,MAAM,WAAW,sBAAsB,CAAC,KAAK,GAAG,OAAO;IACrD,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,oBAAoB,CAAA;IAC3B,KAAK,EAAE,KAAK,CAAA;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAClC,UAAU,EAAE,mBAAmB,EAAE,CAAA;IACjC,UAAU,EAAE,wBAAwB,EAAE,CAAA;IACtC,KAAK,EAAE,4BAA4B,CAAA;IACnC,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,kBAAkB,CAAC,KAAK,GAAG,OAAO;IACjD,EAAE,EAAE,MAAM,CAAA;IACV,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,sBAAsB,CAAC,KAAK,CAAC,EAAE,CAAA;IACtC,KAAK,EAAE,oBAAoB,CAAA;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,yBAAyB,CAAC,KAAK,GAAG,OAAO,CAAE,SAAQ,2BAA2B;IAC7F,OAAO,EAAE,0BAA0B,CAAC,KAAK,CAAC,CAAA;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAClC,KAAK,CAAC,EAAE,uBAAuB,CAAC,KAAK,CAAC,CAAA;IACtC,WAAW,CAAC,EAAE,WAAW,CAAA;IACzB,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,uBAAuB,CAAC,KAAK,GAAG,OAAO;IACtD,IAAI,CAAC,GAAG,EAAE,kBAAkB,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACnD,IAAI,IAAI,OAAO,CAAC,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;CAC7C;AAKD,wBAAsB,kBAAkB,CAAC,KAAK,GAAG,OAAO,EACtD,KAAK,EAAE,mBAAmB,CAAC,KAAK,CAAC,EAAE,EACnC,OAAO,EAAE,yBAAyB,CAAC,KAAK,CAAC,GACxC,OAAO,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CA4FpC;AAED,wBAAgB,iCAAiC,CAC/C,QAAQ,EAAE,kBAAkB,EAC5B,SAAS,EAAE,kBAAkB,EAC7B,MAAM,GAAE,8BAAmC,GAC1C,gCAAgC,CAElC;AAED,wBAAgB,4BAA4B,CAAC,KAAK,GAAG,OAAO,EAC1D,OAAO,GAAE,kBAAkB,CAAC,KAAK,CAAC,EAAO,GACxC,uBAAuB,CAAC,KAAK,CAAC,CAUhC;AAkBD,wBAAgB,yBAAyB,CAAC,KAAK,GAAG,OAAO,EACvD,IAAI,EAAE,MAAM,GACX,uBAAuB,CAAC,KAAK,CAAC,CAgBhC;AAED,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,uBAAuB,EAAE,EACpC,OAAO,GAAE,2BAAgC,GACxC,oBAAoB,CAkBtB;AAED,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,oBAAoB,EAC9B,SAAS,EAAE,oBAAoB,GAC9B,8BAA8B,EAAE,CAkBlC;AAED,wBAAgB,8BAA8B,CAC5C,QAAQ,EAAE,oBAAoB,EAC9B,SAAS,EAAE,oBAAoB,EAC/B,MAAM,GAAE,8BAAmC,GAC1C,gCAAgC,CAuElC;AAED,wBAAgB,6BAA6B,CAC3C,SAAS,EAAE,mBAAmB,EAC9B,SAAS,EAAE,wBAAwB,GAClC,oBAAoB,CAgBtB"}

package/dist/reflective-mutation.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Reflective mutation — primitives for trace-conditioned prompt rewriting.
+ *
+ * Used by `prompt-evolution.ts` (and any consumer running iterative
+ * improvement). Given a parent prompt + concrete trace evidence (top trials,
+ * bottom trials, missed expectations), produce an LLM-ready prompt that
+ * proposes targeted mutations — not blind rephrasings.
+ *
+ * Why this lives outside `prompt-evolution.ts`: any consumer that wants to
+ * run reflective rewriting WITHOUT the population/Pareto machinery can
+ * import these primitives directly.
+ *
+ * Quality bar (vs. naive "mutate this prompt"):
+ *   - Show parent ↔ children diff, not just one variant
+ *   - Quote specific missed goldens with their match phrases
+ *   - Surface the model's actual emitted output side-by-side with what was expected
+ *   - Quote concrete mutation primitives so the model has a vocabulary
+ */
+export interface TrialTrace {
+    /** Stable id for the trial — surfaces in the prompt for grounding. */
+    id: string;
+    /** Score the trial received on its primary metric. */
+    score: number;
+    /** Candidate inputs the agent was given (e.g., the fixture or scenario). */
+    inputName?: string;
+    /**
+     * Goldens / expectations this trial was tested against, with whether each
+     * was matched. The reflection prompt quotes the missed ones specifically.
+     */
+    expectations?: Array<{
+        id: string;
+        phrase: string;
+        matched: boolean;
+    }>;
+    /** Free-form text — what the agent actually emitted (e.g., findings, plan). */
+    emitted?: string;
+    /** The judge's free-form note on WHY this trial scored low — the failure
+     *  evidence the reflection targets. Should be a generalizable pattern (which
+     *  checks/lines/dimensions failed, and how), never case-specific ground
+     *  truth (that would teach memorization, not capability). */
+    failureNote?: string;
+    /** Optional structured metrics (recall, precision, cost, latency). */
+    metrics?: Record<string, number>;
+}
+export interface ReflectionContext {
+    /** What is being mutated — appears in the system prompt for orientation. */
+    target: string;
+    /** Current variant's payload — JSON-serialised for the prompt. */
+    parentPayload: unknown;
+    /** Best-performing trials this generation. */
+    topTrials: TrialTrace[];
+    /** Worst-performing trials this generation — the missed-golden source. */
+    bottomTrials: TrialTrace[];
+    /** How many children the mutator should propose. */
+    childCount: number;
+    /** Optional: domain-specific mutation primitives the model can pick from. */
+    mutationPrimitives?: string[];
+}
+export declare const DEFAULT_MUTATION_PRIMITIVES: string[];
+/**
+ * Build the LLM-ready reflection prompt. Output is plain text — pass it as
+ * the user message. The system message should be small and stable (e.g.
+ * "Output ONLY a JSON object matching the schema below.").
+ */
+export declare function buildReflectionPrompt(ctx: ReflectionContext): string;
+/** Render analyst findings + an optional analysis report into a prompt evidence
+ *  block. A findings producer diagnoses the generation's traces; this surfaces
+ *  that diagnosis to the proposer so a mutation targets a NAMED root cause instead of blindly
+ *  rephrasing. Duck-types the finding shape (claim / severity / area /
+ *  recommended_action) so any findings producer's rows render. Returns null when
+ *  there is nothing to add, so callers append nothing. */
+export declare function renderAnalystEvidence(findings: unknown[], report?: unknown): string | null;
+export interface ReflectionProposal {
+    label: string;
+    rationale: string;
+    payload: unknown;
+}
+export declare function parseReflectionResponse(raw: string, maxProposals?: number): ReflectionProposal[];
+//# sourceMappingURL=reflective-mutation.d.ts.map

package/dist/reflective-mutation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reflective-mutation.d.ts","sourceRoot":"","sources":["../src/reflective-mutation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,MAAM,WAAW,UAAU;IACzB,sEAAsE;IACtE,EAAE,EAAE,MAAM,CAAA;IACV,sDAAsD;IACtD,KAAK,EAAE,MAAM,CAAA;IACb,4EAA4E;IAC5E,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;;OAGG;IACH,YAAY,CAAC,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACtE,+EAA+E;IAC/E,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;iEAG6D;IAC7D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,sEAAsE;IACtE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CACjC;AAED,MAAM,WAAW,iBAAiB;IAChC,4EAA4E;IAC5E,MAAM,EAAE,MAAM,CAAA;IACd,kEAAkE;IAClE,aAAa,EAAE,OAAO,CAAA;IACtB,8CAA8C;IAC9C,SAAS,EAAE,UAAU,EAAE,CAAA;IACvB,0EAA0E;IAC1E,YAAY,EAAE,UAAU,EAAE,CAAA;IAC1B,oDAAoD;IACpD,UAAU,EAAE,MAAM,CAAA;IAClB,6EAA6E;IAC7E,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAA;CAC9B;AAED,eAAO,MAAM,2BAA2B,EAAE,MAAM,EAO/C,CAAA;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,iBAAiB,GAAG,MAAM,CAqFpE;AAED;;;;;0DAK0D;AAC1D,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAuC1F;AAWD,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,OAAO,CAAA;CACjB;AA0DD,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,kBAAkB,EAAE,CA8EhG"}

package/dist/registry.d.ts

@@ -0,0 +1,31 @@
+import type { Scenario, ScenarioFile } from './types';
+/**
+ * ScenarioRegistry — manages scenario discovery and filtering.
+ *
+ * Each agent registers its scenarios. The registry handles conversion
+ * from ScenarioFile format to the framework's Scenario type.
+ */
+export declare class ScenarioRegistry {
+    private scenarios;
+    private scenarioFiles;
+    /** Register scenarios from ScenarioFile format */
+    registerFiles(files: ScenarioFile[]): void;
+    /** Register pre-built Scenario objects directly */
+    register(scenarios: Scenario[]): void;
+    /** Get all scenarios */
+    all(): Scenario[];
+    /** Get scenarios filtered by category */
+    byCategory(category: string): Scenario[];
+    /** List all categories with counts */
+    listCategories(): {
+        category: string;
+        count: number;
+    }[];
+    /** Get scenarios filtered by persona */
+    byPersona(persona: string): Scenario[];
+    /** Get a single scenario by ID */
+    byId(id: string): Scenario | undefined;
+    /** Count total scenarios */
+    get count(): number;
+}
+//# sourceMappingURL=registry.d.ts.map

package/dist/registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAErD;;;;;GAKG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,SAAS,CAAiB;IAClC,OAAO,CAAC,aAAa,CAAqB;IAE1C,kDAAkD;IAClD,aAAa,CAAC,KAAK,EAAE,YAAY,EAAE,GAAG,IAAI;IAK1C,mDAAmD;IACnD,QAAQ,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,IAAI;IAIrC,wBAAwB;IACxB,GAAG,IAAI,QAAQ,EAAE;IAIjB,yCAAyC;IACzC,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,EAAE;IAKxC,sCAAsC;IACtC,cAAc,IAAI;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE;IAQvD,wCAAwC;IACxC,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,QAAQ,EAAE;IAItC,kCAAkC;IAClC,IAAI,CAAC,EAAE,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS;IAItC,4BAA4B;IAC5B,IAAI,KAAK,IAAI,MAAM,CAElB;CACF"}

package/dist/release-confidence.d.ts

@@ -0,0 +1,128 @@
+/**
+ * Release confidence gate.
+ *
+ * This is the production-facing composition layer over the lower-level
+ * primitives:
+ *   - Dataset manifests prove corpus/version coverage.
+ *   - RunRecord rows prove reproducible search/holdout outcomes.
+ *   - Multi-shot trace evidence carries turn counts and ASI diagnostics.
+ *   - HeldOutGate decisions remain the paired promotion authority.
+ *
+ * The gate is intentionally pure and conservative. Missing declared evidence
+ * fails closed instead of being treated as a neutral zero.
+ */
+import type { DatasetManifest, DatasetScenario, DatasetSplit } from './dataset';
+import type { GateDecision } from './held-out-gate';
+import type { RunRecord, RunSplitTag } from './run-record';
+/** Severity of an actionable finding attached to a run/trace. */
+export type AsiSeverity = 'info' | 'warning' | 'error' | 'critical';
+/** Actionable side-info — a diagnosed finding the loop can act on. */
+export interface ActionableSideInfo {
+    /** Stable expectation/check id when available. */
+    expectationId?: string;
+    /** Human-readable diagnosis of what happened. */
+    message: string;
+    severity?: AsiSeverity;
+    /** Concrete trace excerpt, file path, tool call, screenshot id, etc. */
+    evidence?: string;
+    /** Prompt/tool/context surface likely responsible. */
+    responsibleSurface?: string;
+    /** Suggested fix in natural language. */
+    suggestion?: string;
+    /** Whether this expectation was satisfied. Defaults to false for ASI rows. */
+    matched?: boolean;
+    metadata?: Record<string, unknown>;
+}
+export type ReleaseConfidenceStatus = 'pass' | 'warn' | 'fail';
+export type ReleaseConfidenceAxisName = 'corpus' | 'quality' | 'generalization' | 'diagnostics' | 'efficiency';
+export interface ReleaseTraceEvidence {
+    scenarioId: string;
+    candidateId?: string;
+    split?: RunSplitTag;
+    score?: number;
+    ok?: boolean;
+    turnCount?: number;
+    costUsd?: number;
+    durationMs?: number;
+    failureMode?: string;
+    asi?: ActionableSideInfo[];
+    metadata?: Record<string, unknown>;
+}
+export interface ReleaseConfidenceThresholds {
+    /** Require a Dataset manifest or explicit scenarios. Default true. */
+    requireCorpus?: boolean;
+    minScenarioCount?: number;
+    minSearchRuns?: number;
+    minHoldoutRuns?: number;
+    /** Require at least one holdout scenario/run. Default true. */
+    requireHoldout?: boolean;
+    minPassRate?: number;
+    minMeanScore?: number;
+    /** Search mean may exceed holdout mean by at most this much. */
+    maxOverfitGap?: number;
+    maxMeanCostUsd?: number;
+    maxP95WallMs?: number;
+    /** Low-score/failed rows must carry ASI. Default true. */
+    requireAsiForFailures?: boolean;
+    /** Score below this is considered a failure for ASI coverage. Default 0.5. */
+    failureScoreThreshold?: number;
+}
+export interface ReleaseConfidenceInput {
+    target: string;
+    candidateId?: string;
+    baselineId?: string;
+    dataset?: DatasetManifest;
+    scenarios?: readonly DatasetScenario[];
+    runs?: readonly RunRecord[];
+    traces?: readonly ReleaseTraceEvidence[];
+    gateDecision?: GateDecision | null;
+    thresholds?: ReleaseConfidenceThresholds;
+}
+export interface ReleaseConfidenceAxis {
+    name: ReleaseConfidenceAxisName;
+    status: ReleaseConfidenceStatus;
+    score: number;
+    detail: string;
+}
+export interface ReleaseConfidenceIssue {
+    axis: ReleaseConfidenceAxisName;
+    severity: 'critical' | 'warning';
+    code: string;
+    detail: string;
+}
+export interface ReleaseConfidenceMetrics {
+    scenarioCount: number;
+    searchRuns: number;
+    holdoutRuns: number;
+    passRate: number;
+    meanScore: number;
+    searchMeanScore: number;
+    holdoutMeanScore: number;
+    overfitGap: number;
+    meanCostUsd: number;
+    p95WallMs: number;
+    failedRows: number;
+    failuresWithAsi: number;
+    singleShotTraces: number;
+    multiShotTraces: number;
+    splitCounts: Record<DatasetSplit, number>;
+    domainCounts: Record<string, number>;
+    failureModeCounts: Record<string, number>;
+    responsibleSurfaceCounts: Record<string, number>;
+}
+export interface ReleaseConfidenceScorecard {
+    target: string;
+    candidateId: string | null;
+    baselineId: string | null;
+    status: ReleaseConfidenceStatus;
+    promote: boolean;
+    axes: ReleaseConfidenceAxis[];
+    issues: ReleaseConfidenceIssue[];
+    metrics: ReleaseConfidenceMetrics;
+    dataset: DatasetManifest | null;
+    gateDecision: GateDecision | null;
+    summary: string;
+}
+export declare function evaluateReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard;
+export declare function assertReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard;
+//# sourceMappingURL=release-confidence.d.ts.map

package/dist/release-confidence.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"release-confidence.d.ts","sourceRoot":"","sources":["../src/release-confidence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AAE/E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAA;AAE1D,iEAAiE;AACjE,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,CAAA;AAEnE,sEAAsE;AACtE,MAAM,WAAW,kBAAkB;IACjC,kDAAkD;IAClD,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,iDAAiD;IACjD,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,WAAW,CAAA;IACtB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,sDAAsD;IACtD,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,yCAAyC;IACzC,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,8EAA8E;IAC9E,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,MAAM,uBAAuB,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,CAAA;AAC9D,MAAM,MAAM,yBAAyB,GACjC,QAAQ,GACR,SAAS,GACT,gBAAgB,GAChB,aAAa,GACb,YAAY,CAAA;AAEhB,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,KAAK,CAAC,EAAE,WAAW,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,EAAE,CAAC,EAAE,OAAO,CAAA;IACZ,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,GAAG,CAAC,EAAE,kBAAkB,EAAE,CAAA;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,2BAA2B;IAC1C,sEAAsE;IACtE,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,+DAA+D;IAC/D,cAAc,CAAC,EAAE,OAAO,CAAA;IACxB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,gEAAgE;IAChE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,0DAA0D;IAC1D,qBAAqB,CAAC,EAAE,OAAO,CAAA;IAC/B,8EAA8E;IAC9E,qBAAqB,CAAC,EAAE,MAAM,CAAA;CAC/B;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,OAAO,CAAC,EAAE,eAAe,CAAA;IACzB,SAAS,CAAC,EAAE,SAAS,eAAe,EAAE,CAAA;IACtC,IAAI,CAAC,EAAE,SAAS,SAAS,EAAE,CAAA;IAC3B,MAAM,CAAC,EAAE,SAAS,oBAAoB,EAAE,CAAA;IACxC,YAAY,CAAC,EAAE,YAAY,GAAG,IAAI,CAAA;IAClC,UAAU,CAAC,EAAE,2BAA2B,CAAA;CACzC;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,yBAAyB,CAAA;IAC/B,MAAM,EAAE,uBAAuB,CAAA;IAC/B,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,yBAAyB,CAAA;IAC/B,QAAQ,EAAE,UAAU,GAAG,SAAS,CAAA;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,wBAAwB;IACvC,aAAa,EAAE,MAAM,CAAA;IACrB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,eAAe,EAAE,MAAM,CAAA;IACvB,gBAAgB,EAAE,MAAM,CAAA;IACxB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,eAAe,EAAE,MAAM,CAAA;IACvB,gBAAgB,EAAE,MAAM,CAAA;IACxB,eAAe,EAAE,MAAM,CAAA;IACvB,WAAW,EAAE,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IACzC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACpC,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACzC,wBAAwB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CACjD;AAED,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;IAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;IACzB,MAAM,EAAE,uBAAuB,CAAA;IAC/B,OAAO,EAAE,OAAO,CAAA;IAChB,IAAI,EAAE,qBAAqB,EAAE,CAAA;IAC7B,MAAM,EAAE,sBAAsB,EAAE,CAAA;IAChC,OAAO,EAAE,wBAAwB,CAAA;IACjC,OAAO,EAAE,eAAe,GAAG,IAAI,CAAA;IAC/B,YAAY,EAAE,YAAY,GAAG,IAAI,CAAA;IACjC,OAAO,EAAE,MAAM,CAAA;CAChB;AAiBD,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,sBAAsB,GAC5B,0BAA0B,CAyE5B;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,sBAAsB,GAAG,0BAA0B,CAMjG"}

package/dist/release-report.d.ts

@@ -0,0 +1,11 @@
+import type { ReleaseConfidenceScorecard } from './release-confidence';
+import type { RunRecord } from './run-record';
+export interface RenderReleaseReportOptions {
+    title?: string;
+    runs?: readonly RunRecord[];
+    comparator?: string;
+    traceAnalystFindings?: readonly string[];
+    nextActions?: readonly string[];
+}
+export declare function renderReleaseReport(scorecard: ReleaseConfidenceScorecard, options?: RenderReleaseReportOptions): string;
+//# sourceMappingURL=release-report.d.ts.map

package/dist/release-report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"release-report.d.ts","sourceRoot":"","sources":["../src/release-report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAA;AACtE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAG7C,MAAM,WAAW,0BAA0B;IACzC,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,IAAI,CAAC,EAAE,SAAS,SAAS,EAAE,CAAA;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,oBAAoB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACxC,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CAChC;AAED,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,0BAA0B,EACrC,OAAO,GAAE,0BAA+B,GACvC,MAAM,CAkFR"}

package/dist/replay.d.ts

@@ -0,0 +1,120 @@
+/**
+ * Replay-from-raw-events — turn every captured campaign run into a
+ * re-runnable artifact.
+ *
+ * `RawProviderSink` captures every provider HTTP envelope; `runEvalCampaign`
+ * makes that capture the default. Together they make every past run a
+ * complete fingerprint of what happened on the wire — enough to replay
+ * the run without burning new LLM cost.
+ *
+ * Three use cases this primitive enables:
+ *
+ *   1. **Post-hoc judging** — apply a new judge / rubric / scoring callback
+ *      to last week's runs without re-calling any LLM. The cost of trying
+ *      a new rubric drops from "another full sweep" to a CPU-bound replay.
+ *   2. **Determinism audits** — replay the same campaign and verify the
+ *      raw responses match byte-for-byte. Any drift is a non-determinism
+ *      bug (in the harness, the prompt builder, the sandbox, …).
+ *   3. **Free judge calibration** — run two judges on identical responses
+ *      and measure inter-judge agreement without doubling LLM spend.
+ *
+ * The interface is deliberately fetch-shaped. Inject `createReplayFetch`
+ * into `LlmClientOptions.fetch` and every `callLlm` transparently reads
+ * from the cache instead of calling the network. No new code path through
+ * the LLM client is needed; the cache hit is invisible to the runner.
+ */
+import { ReplayError } from './errors';
+import type { RawProviderEvent, RawProviderSink } from './trace/raw-provider-sink';
+export declare class ReplayCacheMissError extends ReplayError {
+    readonly url: string;
+    readonly requestKey: string;
+    constructor(url: string, requestKey: string, message?: string);
+}
+export interface ReplayCacheEntry {
+    request: RawProviderEvent;
+    response: RawProviderEvent;
+}
+export interface ReplayCacheStats {
+    total: number;
+    byProvider: Record<string, number>;
+    byModel: Record<string, number>;
+    /** Spans for which we have a request but no response (run aborted mid-call). */
+    orphanRequests: number;
+}
+/**
+ * In-memory deterministic cache of (request → response) keyed on a stable
+ * hash of the request body. Built from a `RawProviderSink` containing
+ * paired `request` and `response` events from a previous run.
+ *
+ * The cache is the source of truth for replay; `createReplayFetch` is a
+ * thin wrapper that reads from it.
+ */
+export declare class ReplayCache {
+    private byKey;
+    private orphans;
+    private byProvider;
+    private byModel;
+    /**
+     * Build a cache from a sink's events. The sink must implement `list()`.
+     * Filter by `runId` / `spanId` to scope to a specific replay.
+     */
+    static fromSink(sink: RawProviderSink, filter?: {
+        runId?: string;
+        spanId?: string;
+    }): Promise<ReplayCache>;
+    /** Build a cache from an in-memory event list. */
+    static fromEvents(events: RawProviderEvent[]): Promise<ReplayCache>;
+    /** Number of cacheable (request, response) pairs in the cache. */
+    size(): number;
+    stats(): ReplayCacheStats;
+    /** Iterate every cached `(request, response)` pair in insertion order. */
+    entries(): IterableIterator<ReplayCacheEntry>;
+    /**
+     * Look up a cached response by hashing the (model, messages, temperature,
+     * maxTokens, response_format) shape. Returns `undefined` on miss; the
+     * caller decides whether to throw, fall back to the network, or skip.
+     */
+    lookup(requestBody: unknown): Promise<ReplayCacheEntry | undefined>;
+}
+export interface ReplayFetchOptions {
+    /**
+     * Behaviour on cache miss. Default `'throw'`. `'fallback'` calls the
+     * `fallbackFetch` (typically `globalThis.fetch`) so a partial replay can
+     * still complete; `'fail-closed'` returns a synthetic 599 response so the
+     * call site sees a non-retriable failure.
+     */
+    onMiss?: 'throw' | 'fallback' | 'fail-closed';
+    fallbackFetch?: typeof fetch;
+    /** Optional callback fired once per replayed call (for telemetry / counters). */
+    onHit?: (info: {
+        url: string;
+        provider: string;
+        model: string;
+    }) => void;
+    /** Optional callback fired on cache miss before the `onMiss` policy applies. */
+    onMissNotify?: (info: {
+        url: string;
+        requestBody: unknown;
+    }) => void;
+}
+/**
+ * Build a `fetch`-shaped function that serves cached responses out of a
+ * `ReplayCache` for any URL ending in `/chat/completions`. Pass through
+ * `LlmClientOptions.fetch` and `callLlm` becomes free.
+ *
+ * Non-`/chat/completions` URLs are passed straight to the fallback fetch
+ * (default: `globalThis.fetch`). This matters because non-LLM HTTP work
+ * (judge HTTP servers, sandbox callbacks) sometimes flows through the same
+ * `fetch` and shouldn't be intercepted.
+ */
+export declare function createReplayFetch(cache: ReplayCache, opts?: ReplayFetchOptions): typeof fetch;
+/**
+ * Convenience iterator over `(request, response)` pairs in a sink — for
+ * post-hoc scoring that doesn't need a `fetch` shim. The judge or scorer
+ * runs purely in-process over cached LLM outputs.
+ */
+export declare function iterateRawCalls(sink: RawProviderSink, filter?: {
+    runId?: string;
+    spanId?: string;
+}): AsyncGenerator<ReplayCacheEntry>;
+//# sourceMappingURL=replay.d.ts.map

package/dist/replay.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../src/replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAA;AAEtC,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAElF,qBAAa,oBAAqB,SAAQ,WAAW;aAEjC,GAAG,EAAE,MAAM;aACX,UAAU,EAAE,MAAM;gBADlB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,EAClC,OAAO,CAAC,EAAE,MAAM;CAInB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,gBAAgB,CAAA;IACzB,QAAQ,EAAE,gBAAgB,CAAA;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAClC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC/B,gFAAgF;IAChF,cAAc,EAAE,MAAM,CAAA;CACvB;AAED;;;;;;;GAOG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,KAAK,CAAsC;IACnD,OAAO,CAAC,OAAO,CAAI;IACnB,OAAO,CAAC,UAAU,CAA6B;IAC/C,OAAO,CAAC,OAAO,CAA6B;IAE5C;;;OAGG;WACU,QAAQ,CACnB,IAAI,EAAE,eAAe,EACrB,MAAM,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAO,GAC/C,OAAO,CAAC,WAAW,CAAC;IAQvB,kDAAkD;WACrC,UAAU,CAAC,MAAM,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC;IA2BzE,kEAAkE;IAClE,IAAI,IAAI,MAAM;IAId,KAAK,IAAI,gBAAgB;IASzB,0EAA0E;IACzE,OAAO,IAAI,gBAAgB,CAAC,gBAAgB,CAAC;IAI9C;;;;OAIG;IACG,MAAM,CAAC,WAAW,EAAE,OAAO,GAAG,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;CAI1E;AAED,MAAM,WAAW,kBAAkB;IACjC;;;;;OAKG;IACH,MAAM,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,aAAa,CAAA;IAC7C,aAAa,CAAC,EAAE,OAAO,KAAK,CAAA;IAC5B,iFAAiF;IACjF,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAA;IACxE,gFAAgF;IAChF,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,OAAO,CAAA;KAAE,KAAK,IAAI,CAAA;CACrE;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,GAAE,kBAAuB,GAAG,OAAO,KAAK,CA+CjG;AAED;;;;GAIG;AACH,wBAAuB,eAAe,CACpC,IAAI,EAAE,eAAe,EACrB,MAAM,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAC/C,cAAc,CAAC,gBAAgB,CAAC,CAOlC"}

package/dist/reporter.d.ts

@@ -0,0 +1,14 @@
+import type { BenchmarkReport, DriverResult } from './types';
+/**
+ * Report generation utilities.
+ *
+ * Outputs convergence curves, cost curves, quality curves,
+ * and per-persona summaries in markdown format.
+ */
+/** Generate a markdown report from benchmark results */
+export declare function formatBenchmarkReport(report: BenchmarkReport): string;
+/** Generate a markdown report from agent driver results */
+export declare function formatDriverReport(results: DriverResult[]): string;
+/** Print a compact summary to console */
+export declare function printDriverSummary(results: DriverResult[]): void;
+//# sourceMappingURL=reporter.d.ts.map

package/dist/reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAE5D;;;;;GAKG;AAEH,wDAAwD;AACxD,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CAwDrE;AAED,2DAA2D;AAC3D,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,MAAM,CA6ClE;AAED,yCAAyC;AACzC,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,IAAI,CAgBhE"}

package/dist/reporting.d.ts

@@ -1,15 +1,15 @@
-export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from './rubric-predictive-validity-Cy_W-hWZ.js';
-export { B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, R as ReleaseConfidenceAxis, b as ReleaseConfidenceAxisName, c as ReleaseConfidenceInput, d as ReleaseConfidenceIssue, e as ReleaseConfidenceMetrics, f as ReleaseConfidenceScorecard, g as ReleaseConfidenceStatus, h as ReleaseConfidenceThresholds, i as ReleaseTraceEvidence, j as RenderReleaseReportOptions, V as Verdict, k as assertReleaseConfidence, l as bootstrapCi, m as evaluateReleaseConfidence, n as judgeReplayGate, r as renderReleaseReport } from './release-report-euXIV_Sk.js';
-export { I as InterimReleaseConfidence, a as InterimReleaseConfidenceInput, P as PairedEvalueOptions, b as PairedEvalueSequence, c as PairedEvalueStep, S as SequentialDecision, e as evaluateInterimReleaseConfidence, p as pairedEvalueSequence } from './sequential-5iSVfzl2.js';
-export { P as PairedBootstrapOptions, a as PairedBootstrapResult, b as benjaminiHochberg, p as pairedBootstrap, w as wilcoxonSignedRank } from './statistics-C7PozGrZ.js';
-export { G as GainDistributionBin, a as GainDistributionFigureSpec, b as GainDistributionOptions, P as ParetoFigureSpec, c as ParetoPoint, R as RESEARCH_REPORT_HARD_PAIR_FLOOR, d as ResearchReport, e as ResearchReportCandidate, f as ResearchReportDecision, g as ResearchReportMethodology, h as ResearchReportOptions, i as ResearchReportRecommendation, S as SummaryTable, j as SummaryTableOptions, k as SummaryTableRow, l as gainHistogram, p as paretoChart, r as researchReport, s as summaryTable } from './summary-report-DGmUucwQ.js';
-import './run-record-e7vj1uZQ.js';
-import './errors-CzMUYo7b.js';
-import './schema-m0gsnbt3.js';
-import './outcome-store-rnXLEqSn.js';
-import './dataset-BbGkaN2I.js';
-import './judge-calibration-0p2QcWNE.js';
-import './types-Croy5h7V.js';
-import '@tangle-network/tcloud';
-import './failure-cluster-CL7IVgkJ.js';
-import './store-CKUAgsJz.js';
+export type { RubricOutcomePair, RubricPredictiveValidityInput, RubricPredictiveValidityReport, RubricRanking, } from './meta-eval/rubric-predictive-validity';
+export { rubricPredictiveValidity } from './meta-eval/rubric-predictive-validity';
+export type { BootstrapOptions, BootstrapResult, JudgeReplayGateArgs, Verdict, } from './promotion-gate';
+export { bootstrapCi, judgeReplayGate, } from './promotion-gate';
+export type { ReleaseConfidenceAxis, ReleaseConfidenceAxisName, ReleaseConfidenceInput, ReleaseConfidenceIssue, ReleaseConfidenceMetrics, ReleaseConfidenceScorecard, ReleaseConfidenceStatus, ReleaseConfidenceThresholds, ReleaseTraceEvidence, } from './release-confidence';
+export { assertReleaseConfidence, evaluateReleaseConfidence } from './release-confidence';
+export type { RenderReleaseReportOptions } from './release-report';
+export { renderReleaseReport } from './release-report';
+export type { InterimReleaseConfidence, InterimReleaseConfidenceInput, PairedEvalueOptions, PairedEvalueSequence, PairedEvalueStep, SequentialDecision, } from './sequential';
+export { evaluateInterimReleaseConfidence, pairedEvalueSequence, } from './sequential';
+export type { PairedBootstrapOptions, PairedBootstrapResult, } from './statistics';
+export { benjaminiHochberg, pairedBootstrap, wilcoxonSignedRank, } from './statistics';
+export type { GainDistributionBin, GainDistributionFigureSpec, GainDistributionOptions, ParetoFigureSpec, ParetoPoint, ResearchReport, ResearchReportCandidate, ResearchReportDecision, ResearchReportMethodology, ResearchReportOptions, ResearchReportRecommendation, SummaryTable, SummaryTableOptions, SummaryTableRow, } from './summary-report';
+export { gainHistogram, paretoChart, RESEARCH_REPORT_HARD_PAIR_FLOOR, researchReport, summaryTable, } from './summary-report';
+//# sourceMappingURL=reporting.d.ts.map

package/dist/reporting.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporting.d.ts","sourceRoot":"","sources":["../src/reporting.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,iBAAiB,EACjB,6BAA6B,EAC7B,8BAA8B,EAC9B,aAAa,GACd,MAAM,wCAAwC,CAAA;AAC/C,OAAO,EAAE,wBAAwB,EAAE,MAAM,wCAAwC,CAAA;AACjF,YAAY,EACV,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EACnB,OAAO,GACR,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACL,WAAW,EACX,eAAe,GAChB,MAAM,kBAAkB,CAAA;AACzB,YAAY,EACV,qBAAqB,EACrB,yBAAyB,EACzB,sBAAsB,EACtB,sBAAsB,EACtB,wBAAwB,EACxB,0BAA0B,EAC1B,uBAAuB,EACvB,2BAA2B,EAC3B,oBAAoB,GACrB,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAE,uBAAuB,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAA;AACzF,YAAY,EAAE,0BAA0B,EAAE,MAAM,kBAAkB,CAAA;AAClE,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AACtD,YAAY,EACV,wBAAwB,EACxB,6BAA6B,EAC7B,mBAAmB,EACnB,oBAAoB,EACpB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,cAAc,CAAA;AACrB,OAAO,EACL,gCAAgC,EAChC,oBAAoB,GACrB,MAAM,cAAc,CAAA;AACrB,YAAY,EACV,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,cAAc,CAAA;AACrB,OAAO,EACL,iBAAiB,EACjB,eAAe,EACf,kBAAkB,GACnB,MAAM,cAAc,CAAA;AACrB,YAAY,EACV,mBAAmB,EACnB,0BAA0B,EAC1B,uBAAuB,EACvB,gBAAgB,EAChB,WAAW,EACX,cAAc,EACd,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,qBAAqB,EACrB,4BAA4B,EAC5B,YAAY,EACZ,mBAAmB,EACnB,eAAe,GAChB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACL,aAAa,EACb,WAAW,EACX,+BAA+B,EAC/B,cAAc,EACd,YAAY,GACb,MAAM,kBAAkB,CAAA"}