@tangle-network/agent-eval 0.100.1 → 0.101.0

package/dist/multishot/index.d.ts

@@ -1,4 +1,4 @@
-import { J as JudgeScore } from '../types-Cv1bo4_a.js';
+import { J as JudgeScore } from '../types-fWqEJm7h.js';
 import { AgentProfile } from '@tangle-network/agent-interface';
 import { M as MatrixResult } from '../types-BUxNaJ8c.js';
 import '../run-record-DEwidcqn.js';

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.100.1",
+    "version": "0.101.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/{pre-registration-DB8oDqZJ.d.ts → pre-registration-BjGZf9YA.d.ts}

@@ -1,7 +1,7 @@
 import { A as AgentEvalError } from './errors-CzMUYo7b.js';
 import { R as RunRecord } from './run-record-DEwidcqn.js';
 import { C as ChatClient } from './types-BEzCBMQD.js';
-import { c as JudgeDimension, S as Scenario, a as JudgeConfig } from './types-Cv1bo4_a.js';
+import { c as JudgeDimension, S as Scenario, a as JudgeConfig } from './types-fWqEJm7h.js';
 import { TCloud } from '@tangle-network/tcloud';
 import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
 

package/dist/product-benchmark/index.d.ts

@@ -0,0 +1,144 @@
+declare const productBenchmarkSplits: readonly ["practice", "dev", "holdout", "safety", "sentinel"];
+type ProductBenchmarkSplit = (typeof productBenchmarkSplits)[number];
+interface ProductBenchmarkRepoRef {
+    readonly url: string;
+    readonly commit: string;
+    readonly branch: string;
+}
+interface ProductBenchmarkSubstrateVersions {
+    readonly agentEval: string;
+    readonly agentRuntime: string;
+    readonly agentInterface: string;
+    readonly sandbox: string;
+    readonly agentBench?: string;
+}
+interface ProductBenchmarkProfileRef {
+    readonly id: string;
+    readonly profileHash: string;
+    readonly agentProfilePath: string;
+}
+interface ProductBenchmarkArm {
+    readonly id: string;
+    readonly profileId: string;
+    readonly mutableSurfaces: readonly string[];
+    readonly policyAxes: Record<string, unknown>;
+}
+interface ProductBenchmarkScenario {
+    readonly id: string;
+    readonly split: ProductBenchmarkSplit;
+    readonly tags: readonly string[];
+    readonly sourceAllowedForSynthesis: boolean;
+}
+interface ProductBenchmarkBudgets {
+    readonly maxUsd: number;
+    readonly maxCells: number;
+    readonly maxWallMs: number;
+}
+interface ProductBenchmarkManifest {
+    readonly schemaVersion: 1;
+    readonly projectId: string;
+    readonly benchmarkId: string;
+    readonly repo: ProductBenchmarkRepoRef;
+    readonly substrate: ProductBenchmarkSubstrateVersions;
+    readonly profiles: readonly ProductBenchmarkProfileRef[];
+    readonly arms: readonly ProductBenchmarkArm[];
+    readonly scenarios: readonly ProductBenchmarkScenario[];
+    readonly budgets: ProductBenchmarkBudgets;
+    readonly expectedArtifactDir: string;
+}
+interface AgentProfileRuntimeReceipt {
+    readonly model: string;
+    readonly harness: string;
+    readonly backend: string;
+    readonly reasoningEffort?: string;
+}
+type RuntimeResolution = AgentProfileRuntimeReceipt;
+interface ProductBenchmarkRecord {
+    readonly schemaVersion: 1;
+    readonly projectId: string;
+    readonly benchmarkId: string;
+    readonly runId: string;
+    readonly scenarioId: string;
+    readonly split: ProductBenchmarkSplit;
+    readonly armId: string;
+    readonly rep: number;
+    readonly agentProfile: {
+        readonly id: string;
+        readonly hash: string;
+        readonly path: string;
+        readonly declared: RuntimeResolution;
+        readonly resolved: RuntimeResolution;
+    };
+    readonly model: {
+        readonly provider: string;
+        readonly id: string;
+    };
+    readonly backend: {
+        readonly kind: string;
+        readonly version: string;
+    };
+    readonly outcome: {
+        readonly pass: boolean;
+        readonly score: number;
+        readonly dimensions: Record<string, number>;
+        readonly failureMode: string | null;
+    };
+    readonly usage: {
+        readonly inputTokens: number;
+        readonly outputTokens: number;
+        readonly costUsd: number;
+        readonly wallMs: number;
+        readonly toolCalls: number;
+    };
+    readonly integrity: {
+        readonly realBackend: boolean;
+        readonly rawCapture: boolean;
+        readonly traceCapture: boolean;
+        readonly noStubRows: boolean;
+        readonly priced: boolean;
+        readonly profileMaterialized: boolean;
+    };
+    readonly artifacts: {
+        readonly records: string;
+        readonly traces: string;
+        readonly raws: string;
+        readonly scores: string;
+        readonly workspace: string;
+    };
+}
+interface ProductBenchmarkRunInput {
+    readonly manifestPath: string;
+    readonly recordsPath: string;
+    readonly artifactRoot?: string;
+    readonly checkArtifacts?: boolean;
+}
+interface ProductBenchmarkValidationReport {
+    readonly manifestPath: string;
+    readonly recordsPath: string;
+    readonly records: number;
+    readonly projects: readonly string[];
+    readonly benchmarks: readonly string[];
+    readonly arms: readonly string[];
+    readonly scenarios: readonly string[];
+    readonly passed: number;
+    readonly failed: number;
+    readonly inputTokens: number;
+    readonly outputTokens: number;
+    readonly costUsd: number;
+    readonly wallMs: number;
+    readonly integrityFailures: readonly string[];
+    readonly missingArtifacts: readonly string[];
+}
+interface ProductBenchmarkArtifactPaths {
+    readonly manifestPath: string;
+    readonly recordsPath: string;
+}
+declare function validateProductBenchmarkManifest(value: unknown): ProductBenchmarkManifest;
+declare function validateProductBenchmarkRecord(value: unknown): ProductBenchmarkRecord;
+declare function productBenchmarkIntegrityFailures(record: ProductBenchmarkRecord): string[];
+declare function readProductBenchmarkRecords(path: string): ProductBenchmarkRecord[];
+declare function readProductBenchmarkManifest(path: string): ProductBenchmarkManifest;
+declare function validateProductBenchmarkRun(input: ProductBenchmarkRunInput): ProductBenchmarkValidationReport;
+declare function findProductBenchmarkArtifacts(runDir: string): ProductBenchmarkArtifactPaths | null;
+
+export { type AgentProfileRuntimeReceipt, type ProductBenchmarkArm, type ProductBenchmarkArtifactPaths, type ProductBenchmarkBudgets, type ProductBenchmarkManifest, type ProductBenchmarkProfileRef, type ProductBenchmarkRecord, type ProductBenchmarkRepoRef, type ProductBenchmarkRunInput, type ProductBenchmarkScenario, type ProductBenchmarkSplit, type ProductBenchmarkSubstrateVersions, type ProductBenchmarkValidationReport, type RuntimeResolution, findProductBenchmarkArtifacts, productBenchmarkIntegrityFailures, productBenchmarkSplits, readProductBenchmarkManifest, readProductBenchmarkRecords, validateProductBenchmarkManifest, validateProductBenchmarkRecord, validateProductBenchmarkRun };

package/dist/product-benchmark/index.js

@@ -0,0 +1,23 @@
+import {
+  findProductBenchmarkArtifacts,
+  productBenchmarkIntegrityFailures,
+  productBenchmarkSplits,
+  readProductBenchmarkManifest,
+  readProductBenchmarkRecords,
+  validateProductBenchmarkManifest,
+  validateProductBenchmarkRecord,
+  validateProductBenchmarkRun
+} from "../chunk-63MBSQTX.js";
+import "../chunk-3BFEG2F6.js";
+import "../chunk-PZ5AY32C.js";
+export {
+  findProductBenchmarkArtifacts,
+  productBenchmarkIntegrityFailures,
+  productBenchmarkSplits,
+  readProductBenchmarkManifest,
+  readProductBenchmarkRecords,
+  validateProductBenchmarkManifest,
+  validateProductBenchmarkRecord,
+  validateProductBenchmarkRun
+};
+//# sourceMappingURL=index.js.map

package/dist/{provenance-B0SZw1z2.d.ts → provenance-DdfmVfqR.d.ts}

@@ -1,9 +1,9 @@
-import { S as Scenario, g as Gate, G as GateResult, h as GateContext, C as CampaignResult, i as Mutator, f as SurfaceProposer, M as MutableSurface, j as GateDecision } from './types-Cv1bo4_a.js';
+import { S as Scenario, g as Gate, G as GateResult, h as GateContext, C as CampaignResult, i as Mutator, f as SurfaceProposer, M as MutableSurface, j as GateDecision } from './types-fWqEJm7h.js';
 import { R as RedTeamCase } from './red-team-BWdoyleI.js';
 import { R as RunRecord } from './run-record-DEwidcqn.js';
 import { D as Direction } from './pareto-E-pembql.js';
 import { a as PairedBootstrapResult } from './statistics-xP-cWc5k.js';
-import { R as RunCampaignOptions, C as CampaignStorage } from './gepa-BRgNnmGZ.js';
+import { R as RunCampaignOptions, C as CampaignStorage } from './gepa-CEy1AIWp.js';
 import { HostedClient, TraceSpanEvent } from './hosted/index.js';
 
 /**

package/dist/rl.d.ts

@@ -10,7 +10,7 @@ import { R as Researcher, F as FailureMode, S as SteeringChange, E as Experiment
 export { r as runEvalCampaign } from './researcher-Ba2y1Foi.js';
 import { a as VerificationReport } from './multi-layer-verifier-DUZXrPDA.js';
 import { I as InterimReleaseConfidence } from './sequential-5iSVfzl2.js';
-import { C as CampaignResult } from './types-Cv1bo4_a.js';
+import { C as CampaignResult } from './types-fWqEJm7h.js';
 import '@tangle-network/agent-interface';
 import './errors-CzMUYo7b.js';
 import './schema-m0gsnbt3.js';

package/dist/{run-campaign-OWCFOEQG.js → run-campaign-HG4WTSDH.js}

@@ -1,10 +1,12 @@
 import {
+  planCampaignRun,
   runCampaign
-} from "./chunk-HRGTA6U5.js";
+} from "./chunk-XIOQHCHU.js";
 import "./chunk-HRGUJTER.js";
 import "./chunk-3BFEG2F6.js";
 import "./chunk-PZ5AY32C.js";
 export {
+  planCampaignRun,
   runCampaign
 };
-//# sourceMappingURL=run-campaign-OWCFOEQG.js.map
+//# sourceMappingURL=run-campaign-HG4WTSDH.js.map

package/dist/run-campaign-HG4WTSDH.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/{types-Cv1bo4_a.d.ts → types-fWqEJm7h.d.ts}

@@ -432,6 +432,9 @@ interface LabeledScenarioStore {
     }>;
 }
 interface CampaignCellResult<TArtifact> {
+    /** Manifest that produced this cell. Resumability refuses to reuse a cell
+     *  whose manifest differs from the current run. */
+    manifestHash?: string;
     cellId: string;
     scenarioId: string;
     rep: number;

package/docs/concepts.md

@@ -18,6 +18,7 @@ can; drop to the raw functions when you need lower-level control.
 |---|---|---|---|
 | **`defineAgentEval()`** | You have scenarios, an agent, a judge, and a baseline surface, and you want one object you can score or improve. | scenarios, agent, judge, baseline surface | `{ evaluate(), improve() }` where `evaluate()` returns a campaign result and `improve()` returns a decision packet |
 | **`selfImprove()`** | You have a closed loop — scenarios, judge, agent in hand, and you want the substrate to propose better candidates + gate them. | scenarios, agent, judge, baseline surface | `SelfImproveResult.insight: InsightReport` + ship/hold verdict + winner surface |
+| **`loadEvalFixtureScenarios()`** | You want agents to add evals as folders with `PROMPT.md`, checks, and starter files. | `evals/<name>/PROMPT.md + EVAL.ts + package.json` | `Scenario[]` that runs through `runCampaign`; pair with `planEvalFixtureRun()` before spending tokens |
 | **`analyzeRuns()`** | You have observed runs (production traces, an approve/reject corpus, a CSV gold set) and want the same rigor packet without invoking an agent. | `RunRecord[]` + optional flags | `InsightReport` |
 | **Intake adapters** (`fromFeedbackTable`, `fromOtelSpans`) | Your data isn't already in `RunRecord` shape — it's in Obsidian, Sheets, an OTel collector, etc. | source-specific input | `RunRecord[]` ready to pipe into `analyzeRuns()` |
 

package/docs/eval-fixtures.md

@@ -0,0 +1,115 @@
+# Eval Fixtures
+
+Use eval fixtures when you want the easiest coding-agent eval shape:
+
+```text
+evals/
+  fix-login/
+    PROMPT.md
+    EVAL.ts
+    package.json
+    src/...
+```
+
+This copies the best Vercel-style DX without adding a second runner. Fixtures
+become `Scenario` rows, `planEvalFixtureRun()` tells you what is cached vs what
+will run, and `runCampaign()` remains the only execution primitive.
+
+## When To Use It
+
+Use fixtures when:
+
+- a human or agent should add a new eval by creating one folder
+- the prompt, starter files, and validation file should fingerprint together
+- you want a dry run before spending model tokens
+- a runtime or sandbox dispatch already knows how to run one scenario
+
+Do not use fixtures to bypass campaign scoring, held-out gates, traces, or
+backend checks. The fixture folder is input data; the dispatch still owns agent
+execution.
+
+## API
+
+```ts
+import {
+  discoverEvalFixtures,
+  loadEvalFixtureScenarios,
+  planEvalFixtureRun,
+  runCampaign,
+} from '@tangle-network/agent-eval/campaign'
+```
+
+| Export | Use |
+| --- | --- |
+| `discoverEvalFixtures(evalsDir)` | Return nested fixture names containing exact-case `PROMPT.md`. |
+| `loadEvalFixture(evalsDir, name)` | Load one fixture, validate `EVAL.ts`/`EVAL.tsx` and `package.json`, compute a file fingerprint. |
+| `loadEvalFixtureScenarios(evalsDir)` | Convert fixtures into `Scenario` rows for `runCampaign`. |
+| `planEvalFixtureRun(opts)` | Preview cached vs runnable fixture cells without invoking the agent. |
+| `planCampaignRun(opts)` | Same preview for any campaign scenario set. |
+
+`validation: 'vitest'` is the default and requires `PROMPT.md`,
+`EVAL.ts` or `EVAL.tsx`, and `package.json` with `"type": "module"`.
+Use `validation: 'none'` for prompt-only corpora.
+
+## Minimal Flow
+
+```ts
+const scenarios = loadEvalFixtureScenarios('evals', {
+  fingerprintConfig: {
+    model: 'claude-sonnet-4-6@2026-06-01',
+    tools: ['shell', 'edit'],
+  },
+})
+
+const plan = planEvalFixtureRun({
+  evalsDir: 'evals',
+  runDir: '.agent-eval/runs/coding',
+  dispatchRef: 'codex/sandbox/v1',
+  fingerprintConfig: {
+    model: 'claude-sonnet-4-6@2026-06-01',
+    tools: ['shell', 'edit'],
+  },
+})
+
+if (plan.cellsToRun > 0) {
+  await runCampaign({
+    scenarios,
+    dispatch,
+    dispatchRef: 'codex/sandbox/v1',
+    judges,
+    runDir: '.agent-eval/runs/coding',
+    expectUsage: 'assert',
+  })
+}
+```
+
+`dispatchRef` is part of the campaign manifest. Change it when the dispatch's
+real behavior changes: model, tool set, sandbox profile, prompt wrapper, or
+runtime strategy. That keeps resume caches honest.
+
+## What Agents Should Extend
+
+Agents should add capability in this order:
+
+1. Add a fixture folder under `evals/<name>/`.
+2. Put the user task in `PROMPT.md`.
+3. Put deterministic checks in `EVAL.ts` when the task has a code artifact.
+4. Add starter files beside it.
+5. Update the dispatch that maps `EvalFixtureScenario` to the real runtime or
+   sandbox run.
+6. Add or tune judges only after deterministic checks exist.
+7. Run `planEvalFixtureRun()` before `runCampaign()` and report the counts.
+
+Do not add a parallel cache. `runCampaign` stores `manifestHash` on each cached
+cell and refuses stale reuse.
+
+## Example
+
+Run the offline quickstart:
+
+```sh
+pnpm tsx examples/eval-fixtures-quickstart/index.ts
+```
+
+The example prints the plan before and after a run so you can see the cache
+transition from `2 to run / 0 cached` to `0 to run / 2 cached`.

package/docs/feature-guide.md

@@ -31,6 +31,7 @@ trying, and whether a change made them better or worse.
 | “Should the agent keep trying?” | `runAgentControlLoop` | Budgeted `observe -> validate -> decide -> act` runtime. |
 | “The agent should propose, verify, review, and revise.” | `runProposeReviewAsControlLoop` | Reusable preset over the generic control loop. |
 | “Human feedback should become reusable eval data.” | `FeedbackTrajectory` | Captures approvals, rejections, edits, choices, metrics, and policy blocks. |
+| “I want Vercel-style eval folders agents can add quickly.” | `loadEvalFixtureScenarios`, `planEvalFixtureRun`, `runCampaign` | Folder-per-eval DX with campaign scoring and cache-safe dry runs. |
 | “Can this action run, or does it need approval?” | `evaluateActionPolicy` | Generic preflight for side effects, budgets, and required evidence. |
 | “I need train/dev/test/holdout examples.” | `Dataset` plus feedback trajectory conversion | Stable splits and contamination control. |
 | “Which prompt or signature wins?” | `runImprovementLoop`, steering optimizers | Runs variants on scenarios and compares scores. |
@@ -156,6 +157,7 @@ Store as `FeedbackTrajectory`, then derive:
 | Feedback data | `FeedbackTrajectory`, stores, converters | Human/environment labels | Domain adapters live in downstream repos. |
 | Action policy | `evaluateActionPolicy` | Approval/budget preflight | Blocks or labels actions before `act()`. |
 | Datasets | `Dataset`, holdout tools, canaries | Train/dev/test/holdout corpora | Keeps optimization honest. |
+| Eval fixtures | `discoverEvalFixtures`, `loadEvalFixtureScenarios`, `planEvalFixtureRun` | Coding-agent eval folders | Vercel-style input shape; still executes through `runCampaign`. |
 | Optimization | `runImprovementLoop`, steering optimizers | Prompt/signature comparison | Use held-out gates before promotion. |
 | Evolution | prompt/code mutators, sandbox pool, telemetry | Autoresearch and mutation loops | Use budgets and lineage; do not run unbounded. |
 | Telemetry | `TraceStore`, OTLP, file sinks | Audit and replay | Treat traces as evidence, not just logs. |
@@ -201,6 +203,8 @@ Downstream integrations provide domain state, policy, tools, and storage.
 
 - `examples/same-sandbox-harness`: one workdir for install/build/test plus
   evidence inspection.
+- `examples/eval-fixtures-quickstart`: Vercel-style fixture folders with a
+  dry-run plan and campaign cache reuse.
 - `examples/multi-shot-optimization`: full-trajectory optimization with a
   holdout gate.
 - `examples/benchmarks`: benchmark adapter contracts and reference wrappers.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.100.1",
+  "version": "0.101.0",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -114,6 +114,11 @@
       "import": "./dist/perf/index.js",
       "default": "./dist/perf/index.js"
     },
+    "./product-benchmark": {
+      "types": "./dist/product-benchmark/index.d.ts",
+      "import": "./dist/product-benchmark/index.js",
+      "default": "./dist/product-benchmark/index.js"
+    },
     "./multishot": {
       "types": "./dist/multishot/index.d.ts",
       "import": "./dist/multishot/index.js",