@tangle-network/agent-eval 0.94.0 → 0.95.0

package/dist/error-count-extractor.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Toolchain error-count extractor.
+ *
+ * Given stderr/stdout from a compiler or test runner, count the number
+ * of reported errors/failures. Patterns are deliberately narrow —
+ * unknown stderr returns `null` rather than zero so callers can
+ * distinguish "no errors" from "different toolchain, couldn't parse".
+ *
+ * All patterns are anchored to the start of a line and use bounded
+ * character classes to avoid catastrophic backtracking on pathological
+ * inputs.
+ *
+ * Add new toolchains by appending to {@link ERROR_COUNT_PATTERNS};
+ * order matters only in the sense that the first matching pattern wins.
+ */
+export interface ErrorCountPattern {
+    /** Stable identifier for logging + tests. */
+    name: string;
+    /** Must be global (`g` flag) — the extractor counts matches. */
+    regex: RegExp;
+    /** Optional post-processing to extract a count from a single captured match. */
+    transform?: (match: RegExpMatchArray) => number;
+}
+export declare const ERROR_COUNT_PATTERNS: ErrorCountPattern[];
+export interface ExtractOptions {
+    /** Restrict to named patterns — default: all patterns. */
+    only?: string[];
+    /** Additional patterns to consider BEFORE the built-in list. */
+    extra?: ErrorCountPattern[];
+}
+export interface ExtractResult {
+    /** Total count of matched errors, or null when no pattern matched. */
+    count: number | null;
+    /** Name of the pattern that matched, or null. */
+    matched: string | null;
+    /** Original matches for callers that want to surface specifics. */
+    samples: string[];
+}
+/**
+ * Try each pattern in order; return the first with matches.
+ *
+ * Returning `null` (instead of zero) on no-match is deliberate — a
+ * callsite that greps for "typescript errors" on cargo output should
+ * NOT treat that as "zero TS errors" because the toolchain is wrong.
+ */
+export declare function extractErrorCount(text: string, opts?: ExtractOptions): ExtractResult;
+//# sourceMappingURL=error-count-extractor.d.ts.map

package/dist/error-count-extractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"error-count-extractor.d.ts","sourceRoot":"","sources":["../src/error-count-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,MAAM,WAAW,iBAAiB;IAChC,6CAA6C;IAC7C,IAAI,EAAE,MAAM,CAAA;IACZ,gEAAgE;IAChE,KAAK,EAAE,MAAM,CAAA;IACb,gFAAgF;IAChF,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,MAAM,CAAA;CAChD;AAED,eAAO,MAAM,oBAAoB,EAAE,iBAAiB,EAkCnD,CAAA;AAED,MAAM,WAAW,cAAc;IAC7B,0DAA0D;IAC1D,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,gEAAgE;IAChE,KAAK,CAAC,EAAE,iBAAiB,EAAE,CAAA;CAC5B;AAED,MAAM,WAAW,aAAa;IAC5B,sEAAsE;IACtE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,iDAAiD;IACjD,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,mEAAmE;IACnE,OAAO,EAAE,MAAM,EAAE,CAAA;CAClB;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,cAAmB,GAAG,aAAa,CAuBxF"}

package/dist/error-count-extractor.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=error-count-extractor.test.d.ts.map

package/dist/error-count-extractor.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"error-count-extractor.test.d.ts","sourceRoot":"","sources":["../src/error-count-extractor.test.ts"],"names":[],"mappings":""}

package/dist/{errors-CzMUYo7b.d.ts → errors.d.ts}

@@ -12,8 +12,8 @@
  * remain plain `Error`s on purpose — they're programmer-mistake assertions,
  * not consumer-catchable contract failures.
  */
-type AgentEvalErrorCode = 'validation' | 'not_found' | 'config' | 'capture_integrity' | 'judge' | 'verification' | 'replay' | 'backend_integrity' | 'profile_matrix';
-declare class AgentEvalError extends Error {
+export type AgentEvalErrorCode = 'validation' | 'not_found' | 'config' | 'capture_integrity' | 'judge' | 'verification' | 'replay' | 'backend_integrity' | 'profile_matrix';
+export declare class AgentEvalError extends Error {
     /** Stable string code. Survives minification; safe to switch on. */
     readonly code: AgentEvalErrorCode;
     constructor(code: AgentEvalErrorCode, message: string, options?: {
@@ -21,19 +21,19 @@ declare class AgentEvalError extends Error {
     });
 }
 /** Caller passed invalid arguments (out of range, mutually-exclusive options, bad shape). */
-declare class ValidationError extends AgentEvalError {
+export declare class ValidationError extends AgentEvalError {
     constructor(message: string, options?: {
         cause?: unknown;
     });
 }
 /** A named resource (run, span, rubric, scenario, dataset row, route) does not exist. */
-declare class NotFoundError extends AgentEvalError {
+export declare class NotFoundError extends AgentEvalError {
     constructor(message: string, options?: {
         cause?: unknown;
     });
 }
 /** Configuration missing or malformed (`HOME` unset, required image not supplied, env var absent). */
-declare class ConfigError extends AgentEvalError {
+export declare class ConfigError extends AgentEvalError {
     constructor(message: string, options?: {
         cause?: unknown;
     });
@@ -43,28 +43,27 @@ declare class ConfigError extends AgentEvalError {
  * raw HTTP capture absent, no LLM spans, route assertion failed, run-end
  * assertion tripped. Block ship on this; do not catch and move on.
  */
-declare class CaptureIntegrityError extends AgentEvalError {
+export declare class CaptureIntegrityError extends AgentEvalError {
     constructor(message: string, options?: {
         cause?: unknown;
     });
 }
 /** A judge call failed in a way that's not retryable: schema parse failure, bad rubric, conflicting dimensions. */
-declare class JudgeError extends AgentEvalError {
+export declare class JudgeError extends AgentEvalError {
     constructor(message: string, options?: {
         cause?: unknown;
     });
 }
 /** A verifier signalled a hard failure (compile, test, schema) — distinct from a low judge score. */
-declare class VerificationError extends AgentEvalError {
+export declare class VerificationError extends AgentEvalError {
     constructor(message: string, options?: {
         cause?: unknown;
     });
 }
 /** Replay cache cannot satisfy a request: miss with no fallback, sink lacks list(), unsupported URL. */
-declare class ReplayError extends AgentEvalError {
+export declare class ReplayError extends AgentEvalError {
     constructor(message: string, options?: {
         cause?: unknown;
     });
 }
-
-export { AgentEvalError as A, CaptureIntegrityError as C, JudgeError as J, NotFoundError as N, ReplayError as R, ValidationError as V, ConfigError as a, type AgentEvalErrorCode as b, VerificationError as c };
+//# sourceMappingURL=errors.d.ts.map

package/dist/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,MAAM,MAAM,kBAAkB,GAC1B,YAAY,GACZ,WAAW,GACX,QAAQ,GACR,mBAAmB,GACnB,OAAO,GACP,cAAc,GACd,QAAQ,GACR,mBAAmB,GACnB,gBAAgB,CAAA;AAEpB,qBAAa,cAAe,SAAQ,KAAK;IACvC,oEAAoE;IACpE,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAA;gBAErB,IAAI,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAKrF;AAED,6FAA6F;AAC7F,qBAAa,eAAgB,SAAQ,cAAc;gBACrC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAG3D;AAED,yFAAyF;AACzF,qBAAa,aAAc,SAAQ,cAAc;gBACnC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAG3D;AAED,sGAAsG;AACtG,qBAAa,WAAY,SAAQ,cAAc;gBACjC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAG3D;AAED;;;;GAIG;AACH,qBAAa,qBAAsB,SAAQ,cAAc;gBAC3C,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAG3D;AAED,mHAAmH;AACnH,qBAAa,UAAW,SAAQ,cAAc;gBAChC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAG3D;AAED,qGAAqG;AACrG,qBAAa,iBAAkB,SAAQ,cAAc;gBACvC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAG3D;AAED,wGAAwG;AACxG,qBAAa,WAAY,SAAQ,cAAc;gBACjC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAG3D"}

package/dist/{researcher-B0C2_fVO.d.ts → eval-campaign.d.ts}

@@ -1,12 +1,3 @@
-import { a as RunSplitTag, b as RunTokenUsage, c as RunJudgeMetadata, J as JudgeScoresRecord, A as AgentProfileCell, d as AgentProfileCellInput, R as RunRecord } from './run-record-e7vj1uZQ.js';
-import { L as LlmClientOptions, a as LlmRouteRequirements } from './llm-client-Bj7g0rqu.js';
-import { h as ResearchReportOptions, d as ResearchReport, m as GateDecision } from './summary-report-BDOFevaT.js';
-import { T as TraceEmitter, R as RunCompleteHook } from './emitter-C2rqGH_l.js';
-import { R as RunIntegrityExpectations, a as RunIntegrityReport } from './integrity-D2t12mMw.js';
-import { R as RawProviderSink } from './raw-provider-sink-C46HDghv.js';
-import { F as FailureClass } from './schema-m0gsnbt3.js';
-import { T as TraceStore } from './store-BcFXE6LG.js';
-
 /**
  * EvalCampaign — opinionated matrix runner that wires the four
  * capture-integrity directives by construction.
@@ -46,17 +37,26 @@ import { T as TraceStore } from './store-BcFXE6LG.js';
  *   - Resume from partial state across crashes
  *   - LLM-call retry beyond what `LlmClient` already does
  */
-
-interface CampaignVariant<V> {
+import { type AgentProfileCell, type AgentProfileCellInput } from './agent-profile-cell';
+import { type LlmClientOptions, type LlmRouteRequirements } from './llm-client';
+import type { JudgeScoresRecord, RunJudgeMetadata, RunRecord, RunSplitTag, RunTokenUsage } from './run-record';
+import { type ResearchReport, type ResearchReportOptions } from './summary-report';
+import type { RunCompleteHook } from './trace/emitter';
+import { TraceEmitter } from './trace/emitter';
+import { type RunIntegrityExpectations, type RunIntegrityReport } from './trace/integrity';
+import { type RawProviderSink } from './trace/raw-provider-sink';
+import type { FailureClass } from './trace/schema';
+import type { TraceStore } from './trace/store';
+export interface CampaignVariant<V> {
     id: string;
     payload: V;
 }
-interface CampaignScenario {
+export interface CampaignScenario {
     scenarioId: string;
     /** Free-form metadata propagated to runs and reports. */
     tags?: Record<string, string>;
 }
-interface CampaignRunContext<V> {
+export interface CampaignRunContext<V> {
     /** Stable run id. The campaign generates this; the runner does not. */
     runId: string;
     /** Logical experiment id (campaignId by default; overridable per-run via opts). */
@@ -83,7 +83,7 @@ interface CampaignRunContext<V> {
      */
     llmOpts: LlmClientOptions;
 }
-interface CampaignRunOutcome {
+export interface CampaignRunOutcome {
     /** Did the run pass? Mirrors `RunOutcome.pass` semantics. */
     pass: boolean;
     /** Score for the run on its split. Maps to `searchScore` or `holdoutScore`. */
@@ -120,9 +120,9 @@ interface CampaignRunOutcome {
      */
     agentProfile?: AgentProfileCell | AgentProfileCellInput;
 }
-type CampaignRunner<V> = (ctx: CampaignRunContext<V>) => Promise<CampaignRunOutcome>;
-type CampaignIntegrityPolicy = 'throw' | 'mark_failed' | 'log';
-interface EvalCampaignOptions<V> {
+export type CampaignRunner<V> = (ctx: CampaignRunContext<V>) => Promise<CampaignRunOutcome>;
+export type CampaignIntegrityPolicy = 'throw' | 'mark_failed' | 'log';
+export interface EvalCampaignOptions<V> {
     /**
      * Stable id for the campaign. Used as the default `experimentId` on
      * every run, and folded into the campaign fingerprint.
@@ -215,14 +215,14 @@ interface EvalCampaignOptions<V> {
         scenarioTags: Record<string, string>;
     }) => AgentProfileCell | AgentProfileCellInput | Promise<AgentProfileCell | AgentProfileCellInput>);
 }
-interface CampaignFactoryParams {
+export interface CampaignFactoryParams {
     campaignId: string;
     runId: string;
     variantId: string;
     scenarioId: string;
     seed: number;
 }
-interface FailedRun {
+export interface FailedRun {
     runId: string;
     variantId: string;
     scenarioId: string;
@@ -230,7 +230,7 @@ interface FailedRun {
     reason: string;
     error?: string;
 }
-interface EvalCampaignResult {
+export interface EvalCampaignResult {
     campaignId: string;
     /** SHA-256 over canonicalised `(variantIds, scenarioIds, seeds, comparator, splitTag, baseUrl, provider, preregistrationHash)`. */
     campaignFingerprint: string;
@@ -245,143 +245,21 @@ interface EvalCampaignResult {
     startedAt: string;
     endedAt: string;
 }
-declare function runEvalCampaign<V>(opts: EvalCampaignOptions<V>): Promise<EvalCampaignResult>;
-
+export declare function runEvalCampaign<V>(opts: EvalCampaignOptions<V>): Promise<EvalCampaignResult>;
 /**
- * Researcher interface — stable hook for an external autonomous-research
- * agent to drive the meta-loop.
+ * Abort a run whose owning work threw or was orphaned by a sibling's genuine
+ * error, finalizing the emitter so hooks fire and the store records a terminal
+ * status. Safe to call unconditionally: it only aborts runs that are still
+ * `running`. Two no-op cases are intentional and benign:
  *
- * Implementations live downstream (typically in a private repo that
- * runs the actual LLM). This package ships only the contract + a
- * `NoopResearcher` so consumers can wire the surface without being
- * forced to implement every method up front.
+ *   - the run was never started (absent from the store) — nothing to finalize
+ *   - the run is already terminal (`completed` / `failed` / `aborted`) —
+ *     finalizing again would overwrite the real outcome (e.g. flip a passed run
+ *     to `aborted` with `{ pass: false, notes: reason }`), so we leave it alone
  *
- * The four methods mirror the four stages of the paper "Two Loops,
- * Three Roles":
- *
- *   inspectFailures   — given the observed runs, what failure modes
- *                       are present? (data → diagnosis)
- *   proposeChange     — given diagnosed failure modes, what
- *                       structural changes should we try?
- *                       (diagnosis → plan delta)
- *   applyChange       — fold the proposed deltas into a concrete
- *                       experiment plan against an existing baseline.
- *                       (plan delta → executable plan)
- *   evaluateChange    — run the plan, return runs + the gate verdict.
- *                       (executable plan → verdict)
- *
- * Composition is the discipline: a Researcher implementation MUST
- * keep these four steps separate and inspectable. Conflating
- * "diagnose + propose + run" into a single LLM call defeats the
- * point of the framework — you can't audit which step lied.
- *
- * THIS INTERFACE IS STABLE. Breaking changes require a new module
- * (e.g. `Researcher2`) so existing implementations keep working.
+ * Any OTHER failure of the store read or the abort write (disk full, FS fault,
+ * backend down) is a genuine diagnostic and propagates rather than being
+ * swallowed.
  */
-
-/** A diagnosed failure mode with the run-IDs that exhibit it. */
-interface FailureMode {
-    /** Short machine-readable code. Must be stable across runs of the
-     *  same researcher to enable longitudinal tracking. */
-    code: string;
-    /** Human-readable description for the paper / dashboard. */
-    description: string;
-    evidence: {
-        /** Run IDs (from `RunRecord.runId`) where this failure mode was
-         *  observed. */
-        runIds: string[];
-        /** Number of run samples that informed the diagnosis. */
-        samples: number;
-    };
-}
-/** A single steering change the researcher wants to try. */
-interface SteeringChange {
-    kind: 'reviewer_prompt' | 'skill_add' | 'skill_remove' | 'threshold' | 'budget';
-    /** Implementation-specific payload. Researcher implementations
-     *  define the schema — keep this `unknown` here to avoid coupling
-     *  the public interface to any one researcher's internal model. */
-    payload: unknown;
-    /** Why the researcher proposed this change. Goes into the audit
-     *  trail next to the failure-mode evidence. */
-    rationale: string;
-    /** Optional self-reported expected delta on the headline metric. */
-    expectedDelta?: number;
-}
-/** A single experiment plan, mapped onto the search/holdout splits. */
-interface ExperimentPlan {
-    baselineCandidateId: string;
-    proposedCandidateId: string;
-    changes: SteeringChange[];
-    /** USD ceiling for the entire experiment. The runner must stop
-     *  before exceeding this and report a partial result. */
-    evaluationBudgetUsd: number;
-    /** Item IDs (your dataset keys) for the search vs holdout splits. */
-    splits: {
-        search: string[];
-        holdout: string[];
-    };
-}
-/** Result of running a plan: every run, plus the gate verdict. */
-interface ExperimentResult {
-    plan: ExperimentPlan;
-    runs: RunRecord[];
-    gateDecision: GateDecision;
-}
-/**
- * The researcher loop. Stable, four-step, inspectable.
- *
- *   ┌──────────┐  inspectFailures  ┌──────────┐  proposeChange ┌──────────┐
- *   │   runs   │ ─────────────────▶│ failures │ ──────────────▶│ changes  │
- *   └──────────┘                   └──────────┘                └────┬─────┘
- *                                                                   │
- *                                                                   ▼
- *                              ┌────────────────┐  applyChange ┌────────┐
- *                              │ ExperimentPlan │ ◀────────────│  base  │
- *                              └────────┬───────┘              └────────┘
- *                                       │
- *                       evaluateChange  ▼
- *                              ┌────────────────┐
- *                              │ ExperimentResult│
- *                              └────────────────┘
- */
-interface Researcher {
-    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
-}
-interface CallbackResearcherOptions {
-    inspectFailures: Researcher['inspectFailures'];
-    proposeChange: Researcher['proposeChange'];
-    applyChange: Researcher['applyChange'];
-    evaluateChange: Researcher['evaluateChange'];
-}
-/**
- * Minimal concrete researcher for tests, scripts, and small integrations.
- * Larger autonomous researchers can still implement `Researcher` directly.
- */
-declare class CallbackResearcher implements Researcher {
-    private readonly callbacks;
-    constructor(callbacks: CallbackResearcherOptions);
-    inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
-}
-/**
- * No-op researcher — fails loud on every method. Use as a placeholder
- * in code paths that wire the interface but don't have an implementation
- * yet. Importantly, this does NOT silently succeed: a no-op researcher
- * that returned empty arrays would muffle the loop's signal that
- * nobody implemented the brain.
- */
-declare class NoopResearcher implements Researcher {
-    private readonly hint;
-    constructor(hint?: string);
-    inspectFailures(_runs: RunRecord[]): Promise<FailureMode[]>;
-    proposeChange(_failures: FailureMode[]): Promise<SteeringChange[]>;
-    applyChange(_changes: SteeringChange[], _baseline: ExperimentPlan): Promise<ExperimentPlan>;
-    evaluateChange(_plan: ExperimentPlan): Promise<ExperimentResult>;
-}
-
-export { CallbackResearcher as C, type ExperimentPlan as E, type FailureMode as F, NoopResearcher as N, type Researcher as R, type SteeringChange as S, type ExperimentResult as a, type EvalCampaignResult as b, type EvalCampaignOptions as c, type CallbackResearcherOptions as d, type CampaignFactoryParams as e, type CampaignIntegrityPolicy as f, type CampaignRunContext as g, type CampaignRunOutcome as h, type CampaignRunner as i, type CampaignScenario as j, type CampaignVariant as k, type FailedRun as l, runEvalCampaign as r };
+export declare function finalizeAbort(emitter: TraceEmitter, runId: string, reason: string): Promise<void>;
+//# sourceMappingURL=eval-campaign.d.ts.map

package/dist/eval-campaign.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-campaign.d.ts","sourceRoot":"","sources":["../src/eval-campaign.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEH,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,qBAAqB,EAG3B,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAkB,KAAK,gBAAgB,EAAE,KAAK,oBAAoB,EAAE,MAAM,cAAc,CAAA;AAE/F,OAAO,KAAK,EACV,iBAAiB,EACjB,gBAAgB,EAEhB,SAAS,EACT,WAAW,EACX,aAAa,EACd,MAAM,cAAc,CAAA;AAErB,OAAO,EAAE,KAAK,cAAc,EAAE,KAAK,qBAAqB,EAAkB,MAAM,kBAAkB,CAAA;AAClG,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EAGL,KAAK,wBAAwB,EAC7B,KAAK,kBAAkB,EACxB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAA6B,KAAK,eAAe,EAAE,MAAM,2BAA2B,CAAA;AAC3F,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAI/C,MAAM,WAAW,eAAe,CAAC,CAAC;IAChC,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,CAAC,CAAA;CACX;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAA;IAClB,yDAAyD;IACzD,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC9B;AAED,MAAM,WAAW,kBAAkB,CAAC,CAAC;IACnC,uEAAuE;IACvE,KAAK,EAAE,MAAM,CAAA;IACb,mFAAmF;IACnF,YAAY,EAAE,MAAM,CAAA;IACpB,OAAO,EAAE,CAAC,CAAA;IACV,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACpC,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,WAAW,CAAA;IACrB;;;;;OAKG;IACH,OAAO,EAAE,YAAY,CAAA;IACrB,KAAK,EAAE,UAAU,CAAA;IACjB,OAAO,EAAE,eAAe,CAAA;IACxB;;;;OAIG;IACH,OAAO,EAAE,gBAAgB,CAAA;CAC1B;AAED,MAAM,WAAW,kBAAkB;IACjC,6DAA6D;IAC7D,IAAI,EAAE,OAAO,CAAA;IACb,+EAA+E;IAC/E,KAAK,EAAE,MAAM,CAAA;IACb,+EAA+E;IAC/E,OAAO,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,aAAa,CAAA;IACzB,+DAA+D;IAC/D,KAAK,EAAE,MAAM,CAAA;IACb,wDAAwD;IACxD,UAAU,EAAE,MAAM,CAAA;IAClB,kFAAkF;IAClF,UAAU,EAAE,MAAM,CAAA;IAClB,+DAA+D;IAC/D,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC5B;;6EAEyE;IACzE,YAAY,CAAC,EAAE,YAAY,CAAA;IAC3B,sEAAsE;IACtE,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,qDAAqD;IACrD,aAAa,CAAC,EAAE,gBAAgB,CAAA;IAChC;;;;OAIG;IACH,WAAW,CAAC,EAAE,iBAAiB,CAAA;IAC/B;;;;OAIG;IACH,YAAY,CAAC,EAAE,gBAAgB,GAAG,qBAAqB,CAAA;CACxD;AAED,MAAM,MAAM,cAAc,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,kBAAkB,CAAC,CAAA;AAE3F,MAAM,MAAM,uBAAuB,GAAG,OAAO,GAAG,aAAa,GAAG,KAAK,CAAA;AAErE,MAAM,WAAW,mBAAmB,CAAC,CAAC;IACpC;;;OAGG;IACH,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,eAAe,CAAC,CAAC,CAAC,EAAE,CAAA;IAC9B,SAAS,EAAE,gBAAgB,EAAE,CAAA;IAC7B,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,sEAAsE;IACtE,QAAQ,CAAC,EAAE,WAAW,CAAA;IACtB,iFAAiF;IACjF,SAAS,EAAE,MAAM,CAAA;IACjB;;;;OAIG;IACH,OAAO,EAAE,gBAAgB,CAAA;IACzB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,oBAAoB,CAAA;IACxC;;;;OAIG;IACH,YAAY,EAAE,CAAC,MAAM,EAAE,qBAAqB,KAAK,UAAU,CAAA;IAC3D;;;;;OAKG;IACH,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,qBAAqB,KAAK,eAAe,CAAA;IACnE;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;OAGG;IACH,aAAa,CAAC,EAAE,eAAe,EAAE,CAAA;IACjC;;;;OAIG;IACH,SAAS,CAAC,EAAE,wBAAwB,CAAA;IACpC,+DAA+D;IAC/D,kBAAkB,CAAC,EAAE,uBAAuB,CAAA;IAC5C;;;OAGG;IACH,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,CAAA;IACzB;;;OAGG;IACH,MAAM,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CACrC,qBAAqB,EACrB,YAAY,GAAG,qBAAqB,GAAG,aAAa,CACrD,CAAA;IACD;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,mDAAmD;IACnD,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,GAAG,CAAC,EAAE,MAAM,MAAM,CAAA;IAClB,oDAAoD;IACpD,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,qBAAqB,KAAK,MAAM,CAAA;IACjD;;;;;OAKG;IACH,YAAY,CAAC,EACT,gBAAgB,GAChB,qBAAqB,GACrB,CAAC,CACC,MAAM,EAAE,qBAAqB,GAAG;QAC9B,OAAO,EAAE,CAAC,CAAA;QACV,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KACrC,KAEC,gBAAgB,GAChB,qBAAqB,GACrB,OAAO,CAAC,gBAAgB,GAAG,qBAAqB,CAAC,CAAC,CAAA;CAC3D;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,CAAA;IAClB,mIAAmI;IACnI,mBAAmB,EAAE,MAAM,CAAA;IAC3B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAA;IAClC,8DAA8D;IAC9D,IAAI,EAAE,SAAS,EAAE,CAAA;IACjB,kDAAkD;IAClD,gBAAgB,EAAE,kBAAkB,EAAE,CAAA;IACtC,UAAU,EAAE,SAAS,EAAE,CAAA;IACvB,gDAAgD;IAChD,MAAM,CAAC,EAAE,cAAc,CAAA;IACvB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;CAChB;AAeD,wBAAsB,eAAe,CAAC,CAAC,EACrC,IAAI,EAAE,mBAAmB,CAAC,CAAC,CAAC,GAC3B,OAAO,CAAC,kBAAkB,CAAC,CA8T7B;AAcD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,YAAY,EACrB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CAKf"}

package/dist/eval-campaign.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=eval-campaign.test.d.ts.map

package/dist/eval-campaign.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-campaign.test.d.ts","sourceRoot":"","sources":["../src/eval-campaign.test.ts"],"names":[],"mappings":""}

package/dist/eval-tools.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Eval primitives as agent tools — `makeEvalTools` packages the substrate's
+ * judge / completion / analysis entry points as JSON-Schema tool definitions
+ * an LLM agent loop can call. The host closes over the live config (judge
+ * panels, the correctness checker, analyst registry); the agent passes only
+ * data over the wire.
+ *
+ * Three tools, each present only when its config section is supplied:
+ *   - `run_judges`        — score an artifact with the configured `JudgeConfig`s
+ *   - `verify_completion` — gold-spec requirement check → `CompletionVerdict`
+ *   - `analyze_runs`      — `RunRecord[]` (inline or from a file) → `InsightReport`
+ *
+ * `toOpenAiTool` converts a definition to the OpenAI function-tool wire shape.
+ */
+import type { JudgeConfig } from './campaign/types';
+import { type CorrectnessChecker } from './completion-verifier';
+import { type AnalyzeRunsOptions } from './contract/analyze-runs';
+/** One agent-callable tool. `parameters` is a JSON Schema (draft-07+) object. */
+export interface EvalToolDef {
+    name: string;
+    description: string;
+    parameters: Record<string, unknown>;
+    handler: (args: unknown, ctx?: {
+        signal?: AbortSignal;
+    }) => Promise<unknown>;
+}
+export interface MakeEvalToolsConfig {
+    /** Judges available to `run_judges`. Omit to exclude the tool. */
+    judges?: Array<JudgeConfig<unknown>>;
+    /** Host-side correctness checker for `verify_completion` (the third
+     *  `verifyCompletion` argument — a function, so it cannot cross the wire).
+     *  Omit to exclude the tool. */
+    completion?: {
+        checkCorrectness: CorrectnessChecker;
+    };
+    /** `analyzeRuns` options minus `runs` (runs arrive as tool args, inline or
+     *  via `path`). Omit to exclude the tool. */
+    analyze?: Omit<AnalyzeRunsOptions, 'runs'>;
+}
+/** OpenAI function-tool wire shape for an `EvalToolDef`. */
+export declare function toOpenAiTool(def: EvalToolDef): {
+    type: 'function';
+    function: {
+        name: string;
+        description: string;
+        parameters: Record<string, unknown>;
+    };
+};
+/**
+ * Build the eval toolset for the supplied config. Only sections present in
+ * `cfg` produce tools, so the agent's tool list mirrors what the host
+ * actually wired. Handlers fail loud on malformed args — no silent defaults.
+ */
+export declare function makeEvalTools(cfg: MakeEvalToolsConfig): EvalToolDef[];
+//# sourceMappingURL=eval-tools.d.ts.map

package/dist/eval-tools.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-tools.d.ts","sourceRoot":"","sources":["../src/eval-tools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAwB,MAAM,kBAAkB,CAAA;AACzE,OAAO,EAEL,KAAK,kBAAkB,EAIxB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,KAAK,kBAAkB,EAAe,MAAM,yBAAyB,CAAA;AAG9E,iFAAiF;AACjF,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACnC,OAAO,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;CAC7E;AAED,MAAM,WAAW,mBAAmB;IAClC,kEAAkE;IAClE,MAAM,CAAC,EAAE,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAA;IACpC;;oCAEgC;IAChC,UAAU,CAAC,EAAE;QAAE,gBAAgB,EAAE,kBAAkB,CAAA;KAAE,CAAA;IACrD;iDAC6C;IAC7C,OAAO,CAAC,EAAE,IAAI,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAA;CAC3C;AAED,4DAA4D;AAC5D,wBAAgB,YAAY,CAAC,GAAG,EAAE,WAAW,GAAG;IAC9C,IAAI,EAAE,UAAU,CAAA;IAChB,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAA;CACrF,CAKA;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,mBAAmB,GAAG,WAAW,EAAE,CAMrE"}

package/dist/eval-trace-store.d.ts

@@ -0,0 +1,107 @@
+/**
+ * EvalTraceStore — JSONL save / query / compare over the analysis-time
+ * `RunRecord` row.
+ *
+ * `FileSystemTraceStore` (in `./trace/store`) persists the rich TraceSchema-v1
+ * span corpus — runs, spans, events, artifacts — the runtime emits live. That
+ * is NOT the thing three consumers hand-roll in `tests/eval/lib/trace-store.ts`.
+ * What they hand-roll is the *analysis* layer: append each finished run as one
+ * JSONL line keyed by scenario/candidate, then query it ("every run where score
+ * < 50"), pick the best run for a scenario (few-shot seeding), and compare two
+ * candidates on matched scenarios.
+ *
+ * The substrate already has the canonical analysis-time row — `RunRecord` — so
+ * this is that query/compare API expressed over `RunRecord[]`, with a
+ * JSONL-backed store. It does NOT fork `FileSystemTraceStore`; it sits beside it
+ * for the analysis projection.
+ */
+import { type RunRecord, type RunSplitTag } from './run-record';
+/** The score the query/compare layer ranks on: holdout when present (the
+ *  gated number), else search. Throws when a record carries neither — a
+ *  RunRecord is invalid without at least one, but a hand-built object might. */
+export declare function runScore(record: RunRecord): number;
+export interface RunRecordFilter {
+    experimentId?: string;
+    candidateId?: string;
+    scenarioId?: string;
+    model?: string;
+    splitTag?: RunSplitTag;
+    /** Inclusive lower bound on `runScore`. */
+    minScore?: number;
+    /** Inclusive upper bound on `runScore`. */
+    maxScore?: number;
+    /** Match a single tag in `outcome.raw` by exact numeric value. */
+    rawEquals?: {
+        key: string;
+        value: number;
+    };
+    /** Custom predicate, ANDed with the structured filters. */
+    where?: (record: RunRecord) => boolean;
+}
+export interface CandidateComparison {
+    a: string;
+    b: string;
+    /** Scenario ids present for BOTH candidates — the paired comparison set. */
+    pairedScenarioIds: string[];
+    /** Mean `runScore` for candidate a over the paired scenarios. */
+    meanA: number;
+    /** Mean `runScore` for candidate b over the paired scenarios. */
+    meanB: number;
+    /** meanB − meanA. Positive ⇒ b scored higher on the matched scenarios. */
+    meanDelta: number;
+    /** Scenarios where b beat a (strictly), tied, and a beat b. */
+    bWins: number;
+    ties: number;
+    aWins: number;
+}
+/**
+ * Backing persistence for `EvalTraceStore`. The in-memory store is the default;
+ * the JSONL file store appends one validated `RunRecord` per line. Both keep an
+ * append order so `getBest` / `compareRuns` are reproducible.
+ */
+export interface RunRecordBackend {
+    append(record: RunRecord): Promise<void>;
+    load(): Promise<RunRecord[]>;
+}
+export declare function inMemoryRunRecordBackend(initial?: RunRecord[]): RunRecordBackend;
+/**
+ * JSONL-backed store at `path`, one `RunRecord` per line. Malformed lines fail
+ * loud on load (a corrupt corpus must not silently shrink the analysis set);
+ * pass `skipInvalid` only for forensics on a known-bad file.
+ */
+export declare function jsonlRunRecordBackend(path: string, opts?: {
+    skipInvalid?: boolean;
+}): RunRecordBackend;
+/**
+ * Query / compare layer over a `RunRecord` corpus. Append finished runs, query
+ * with a structured filter, take the best run for a scenario, and compare two
+ * candidates on their matched scenarios. Persistence is injected via
+ * `RunRecordBackend` (in-memory by default, JSONL file via
+ * `jsonlRunRecordBackend`).
+ */
+export declare class EvalTraceStore {
+    private readonly backend;
+    constructor(backend?: RunRecordBackend);
+    /** Validate and append one run. Throws on an invalid record — the corpus
+     *  stays paper-grade. */
+    append(record: RunRecord): Promise<void>;
+    all(): Promise<RunRecord[]>;
+    query(filter?: RunRecordFilter): Promise<RunRecord[]>;
+    /**
+     * Highest-scoring run for a scenario (optionally restricted to a candidate).
+     * Returns null when no run matches. Ties resolve to the earliest-appended run
+     * so the result is stable.
+     */
+    getBest(scenarioId: string, opts?: {
+        candidateId?: string;
+        splitTag?: RunSplitTag;
+    }): Promise<RunRecord | null>;
+    /**
+     * Compare two candidates on the scenarios they BOTH ran. When a candidate
+     * ran a scenario more than once, its best `runScore` for that scenario is
+     * used. Throws when there is no paired scenario — an unpaired "comparison" is
+     * not one.
+     */
+    compareRuns(candidateA: string, candidateB: string): Promise<CandidateComparison>;
+}
+//# sourceMappingURL=eval-trace-store.d.ts.map

package/dist/eval-trace-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-trace-store.d.ts","sourceRoot":"","sources":["../src/eval-trace-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,EAAe,KAAK,SAAS,EAAE,KAAK,WAAW,EAAqB,MAAM,cAAc,CAAA;AAE/F;;gFAEgF;AAChF,wBAAgB,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,CAOlD;AAED,MAAM,WAAW,eAAe;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,WAAW,CAAA;IACtB,2CAA2C;IAC3C,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,2CAA2C;IAC3C,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,kEAAkE;IAClE,SAAS,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1C,2DAA2D;IAC3D,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,KAAK,OAAO,CAAA;CACvC;AAeD,MAAM,WAAW,mBAAmB;IAClC,CAAC,EAAE,MAAM,CAAA;IACT,CAAC,EAAE,MAAM,CAAA;IACT,4EAA4E;IAC5E,iBAAiB,EAAE,MAAM,EAAE,CAAA;IAC3B,iEAAiE;IACjE,KAAK,EAAE,MAAM,CAAA;IACb,iEAAiE;IACjE,KAAK,EAAE,MAAM,CAAA;IACb,0EAA0E;IAC1E,SAAS,EAAE,MAAM,CAAA;IACjB,+DAA+D;IAC/D,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;CACd;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,CAAC,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACxC,IAAI,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC,CAAA;CAC7B;AAED,wBAAgB,wBAAwB,CAAC,OAAO,GAAE,SAAS,EAAO,GAAG,gBAAgB,CAUpF;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,MAAM,EACZ,IAAI,GAAE;IAAE,WAAW,CAAC,EAAE,OAAO,CAAA;CAAO,GACnC,gBAAgB,CA4ClB;AAED;;;;;;GAMG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;gBAE9B,OAAO,GAAE,gBAA6C;IAIlE;6BACyB;IACnB,MAAM,CAAC,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAIxC,GAAG,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;IAI3B,KAAK,CAAC,MAAM,GAAE,eAAoB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAK/D;;;;OAIG;IACG,OAAO,CACX,UAAU,EAAE,MAAM,EAClB,IAAI,GAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,WAAW,CAAA;KAAO,GAC1D,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IAmB5B;;;;;OAKG;IACG,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC;CAuDxF"}

package/dist/eval-trace-store.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=eval-trace-store.test.d.ts.map

package/dist/eval-trace-store.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-trace-store.test.d.ts","sourceRoot":"","sources":["../src/eval-trace-store.test.ts"],"names":[],"mappings":""}

package/dist/executor.d.ts

@@ -0,0 +1,38 @@
+import type { TCloud } from '@tangle-network/tcloud';
+import type { CollectedArtifacts, JudgeFn, Scenario, ScenarioResult } from './types';
+export interface ExecutorConfig {
+    /** System prompt for the agent under test */
+    systemPrompt: string;
+    /** Model to use for the agent */
+    model?: string;
+    /** Judges to run after execution */
+    judges: JudgeFn[];
+    /** Regex patterns for detecting tool/API calls in responses */
+    toolCallPatterns?: RegExp[];
+    /** Block delimiter pattern (default: :::type\n...\n:::) */
+    blockPattern?: RegExp;
+    /** Custom artifact checker for domain-specific checks */
+    artifactChecker?: (check: Scenario['artifactChecks'][0], artifacts: CollectedArtifacts) => {
+        passed: boolean;
+        detail: string;
+    } | null;
+    /**
+     * Sleep used between judge retries and after a judge succeeds. Defaults to
+     * real `setTimeout`. Injectable so tests exercise the retry policy without
+     * the real multi-second backoff.
+     */
+    sleep?: (ms: number) => Promise<void>;
+}
+/** Per-judge failure diagnostic, recorded additively on `ScenarioResult`. */
+export interface JudgeFailure {
+    judge: string;
+    attempts: number;
+    reason: string;
+}
+/**
+ * Execute a scenario against an LLM via tcloud.
+ *
+ * Runs multi-turn conversation, extracts artifacts, runs judges.
+ */
+export declare function executeScenario(tc: TCloud, scenario: Scenario, config: ExecutorConfig): Promise<ScenarioResult>;
+//# sourceMappingURL=executor.d.ts.map

package/dist/executor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../src/executor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAKpD,OAAO,KAAK,EACV,kBAAkB,EAClB,OAAO,EAEP,QAAQ,EACR,cAAc,EAEf,MAAM,SAAS,CAAA;AAOhB,MAAM,WAAW,cAAc;IAC7B,6CAA6C;IAC7C,YAAY,EAAE,MAAM,CAAA;IACpB,iCAAiC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,oCAAoC;IACpC,MAAM,EAAE,OAAO,EAAE,CAAA;IACjB,+DAA+D;IAC/D,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC3B,2DAA2D;IAC3D,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,yDAAyD;IACzD,eAAe,CAAC,EAAE,CAChB,KAAK,EAAE,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EACpC,SAAS,EAAE,kBAAkB,KAC1B;QAAE,MAAM,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;IAC/C;;;;OAIG;IACH,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CACtC;AAYD,6EAA6E;AAC7E,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;CACf;AAED;;;;GAIG;AACH,wBAAsB,eAAe,CACnC,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,cAAc,GACrB,OAAO,CAAC,cAAc,CAAC,CAoOzB"}

package/dist/executor.test.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Capture-integrity contract for the scenario executor. Two failure classes
+ * that a sloppy fallback erases must stay loud:
+ *   1. a malformed/hung chat response must NOT be recorded as a real empty turn
+ *      (capture defect, indistinguishable from a model that said nothing);
+ *   2. a judge that errors must record WHY, retry only transient faults, and
+ *      fail deterministic faults immediately.
+ */
+export {};
+//# sourceMappingURL=executor.test.d.ts.map

package/dist/executor.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"executor.test.d.ts","sourceRoot":"","sources":["../src/executor.test.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG"}