@tangle-network/agent-eval 0.37.0 → 0.40.1

package/dist/run-campaign-JYJXYHHL.js

@@ -0,0 +1,10 @@
+import {
+  runCampaign
+} from "./chunk-TMXPFWC7.js";
+import "./chunk-WP7SY7AI.js";
+import "./chunk-QYJT52YW.js";
+import "./chunk-PZ5AY32C.js";
+export {
+  runCampaign
+};
+//# sourceMappingURL=run-campaign-JYJXYHHL.js.map

package/dist/run-campaign-JYJXYHHL.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/traces.js

@@ -1,11 +1,9 @@
 import {
-  DEFAULT_REDACTION_RULES,
   DEFAULT_TRACE_ANALYST_BUDGETS,
   FileSystemTraceStore,
   InMemoryTraceStore,
   OTEL_AGENT_EVAL_SCOPE,
   OtlpFileTraceStore,
-  REDACTION_VERSION,
   ReplayCache,
   ReplayCacheMissError,
   SpanNotFoundError,
@@ -30,13 +28,17 @@ import {
   iterateRawCalls,
   otelRunCompleteHook,
   planTraceInsightQuestions,
-  redactString,
-  redactValue,
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystFunctionGroup,
   traceAnalystOnRunComplete
-} from "./chunk-L5UNCDAJ.js";
+} from "./chunk-MAOZCN36.js";
+import {
+  DEFAULT_REDACTION_RULES,
+  REDACTION_VERSION,
+  redactString,
+  redactValue
+} from "./chunk-GGE4NNQT.js";
 import {
   aggregateLlm,
   argHash,

package/dist/types-DHqkLwEU.d.ts

@@ -0,0 +1,110 @@
+import { DefaultVerdict } from '@tangle-network/agent-runtime/loops';
+
+/**
+ * @experimental
+ *
+ * N-axis cartesian matrix over substrate types — types module.
+ *
+ * The matrix is a runner + aggregator. It iterates the cartesian product of
+ * caller-provided axes (any value type — `AgentProfile` from sandbox, `Driver`
+ * / `Validator` from agent-runtime, rubric records, thinking levels, anything)
+ * and aggregates per-axis pass/score/cost summaries. Substrate types are
+ * imported at the boundary by JSDoc only; the matrix never wraps them.
+ */
+
+/** One axis = one dimension to iterate. `V` is the value type — pass any
+ *  substrate type (AgentProfile, Driver, Validator, rubric record). */
+interface MatrixAxis<V> {
+    /** Axis name. Becomes the key in `MatrixResult.byAxis`. */
+    name: string;
+    /** Stable id per value. Used as the bucket key in aggregation. */
+    values: Array<{
+        id: string;
+        value: V;
+    }>;
+    /** Optional bucket label override. Receives the same `(value, id)` the
+     *  runner stored on the cell; default label is `id`. */
+    label?: (value: V, id: string) => string;
+}
+/** A cell carries one picked value from each axis, keyed by axis name. */
+interface MatrixCell {
+    axes: Record<string, {
+        id: string;
+        value: unknown;
+    }>;
+    /** 0-based replicate index within the same axis combination. */
+    rep: number;
+    /** Stable sort key — preserves cartesian order across concurrent execution. */
+    ordinal: number;
+}
+interface CellResult<Output> {
+    output: Output;
+    verdict: DefaultVerdict;
+    costUsd: number;
+    durationMs: number;
+    runId?: string;
+    /** Populated when `runCell` threw. The cell contributes 0 to passRate AND
+     *  meanScore regardless of `verdict`. */
+    error?: {
+        message: string;
+        kind: string;
+    };
+}
+interface AxisSummary {
+    axisName: string;
+    axisValue: string;
+    cells: number;
+    passRate: number;
+    meanScore: number;
+    p50Score: number;
+    p90Score: number;
+    totalCostUsd: number;
+    meanDurationMs: number;
+}
+interface MatrixResult<Output> {
+    cells: Array<{
+        cell: MatrixCell;
+        runs: CellResult<Output>[];
+    }>;
+    /** `byAxis[axisName][axisValueId] = summary`. Populated only for axes
+     *  named in `aggregateBy` (default = every axis in `axes`). */
+    byAxis: Record<string, Record<string, AxisSummary>>;
+    summary: {
+        totalCells: number;
+        runsExecuted: number;
+        /** Cells removed by `filter` plus cells unscheduled after the cost
+         *  ceiling or abort signal tripped. */
+        cellsSkipped: number;
+        overallPassRate: number;
+        overallMeanScore: number;
+        totalCostUsd: number;
+        durationMs: number;
+    };
+    /** Stable id-like string generated at the end of the run. */
+    matrixId: string;
+}
+interface RunAgentMatrixOptions<Output> {
+    axes: MatrixAxis<unknown>[];
+    /** User-supplied cell executor. May throw; the matrix captures throws as
+     *  `CellResult.error` and continues. */
+    runCell: (cell: MatrixCell) => Promise<CellResult<Output>>;
+    /** Replicates per cell. Default 1. */
+    reps?: number;
+    /** Prune cells from the cartesian BEFORE rep expansion. */
+    filter?: (cell: Omit<MatrixCell, 'rep' | 'ordinal'>) => boolean;
+    /** Axes to aggregate into `byAxis`. Default: every axis in `axes`. */
+    aggregateBy?: string[];
+    /** Max concurrent in-flight `runCell` invocations. Default 4. */
+    maxConcurrency?: number;
+    /** Cumulative-cost abort threshold (USD). When the running sum of
+     *  `result.costUsd` crosses this value, no new cells are scheduled.
+     *  In-flight cells finish. Default `Infinity`. */
+    costCeiling?: number;
+    /** Fires once per executed cell, after its promise settles. */
+    onCellComplete?: (cell: MatrixCell, result: CellResult<Output>) => void;
+    /** External cancellation. Aborts in-flight cells via a forwarded signal
+     *  and suppresses scheduling of new ones. */
+    signal?: AbortSignal;
+}
+
+export type { AxisSummary as A, CellResult as C, MatrixResult as M, RunAgentMatrixOptions as R, MatrixAxis as a, MatrixCell as b };

package/dist/wire/index.d.ts

@@ -1,4 +1,4 @@
-import { n as FeedbackTrajectoryStore } from '../feedback-trajectory-iATEAHmc.js';
+import { n as FeedbackTrajectoryStore } from '../feedback-trajectory-Dvy-bt7x.js';
 import { T as TraceStore } from '../store-Db2Bv8Cf.js';
 import { z } from 'zod';
 import { OpenAPIObject } from 'openapi3-ts/oas31';
@@ -7,7 +7,7 @@ import { ServerType } from '@hono/node-server';
 import { Hono } from 'hono';
 import '../control-runtime-BZ_lVLYW.js';
 import '../emitter-DP_cSSiw.js';
-import '../dataset-ueRVTUoY.js';
+import '../dataset-BlwAtYYf.js';
 import '../errors-mje_cKOs.js';
 
 declare const RubricDimensionSchema: z.ZodObject<{

package/docs/design/loop-taxonomy.md

@@ -0,0 +1,233 @@
+# Loop taxonomy: driver, worker, measurement, and the improvement loop
+
+This is the canonical vocabulary for the Tangle agent stack. It exists because
+the same word ("loop", "shot", "worker") was being used at three different
+layers, and the layers were getting conflated. Every role below has exactly
+one meaning. Use these words and nothing else.
+
+Cross-links: [`three-package-architecture.md`](../three-package-architecture.md)
+(who owns what), [`concepts.md`](../concepts.md) (eval mental model),
+[`multi-shot-optimization.md`](../multi-shot-optimization.md) (GEPA),
+[`auto-research-loop-end-to-end.md`](../auto-research-loop-end-to-end.md)
+(analyst / autoresearch).
+
+## The three roles
+
+| Role | Definition | Lives at |
+|---|---|---|
+| **Driver** | The thing that *decides what happens next*. Plans, then decides whether to continue. | Both layers (see below) |
+| **Worker** | An agent harness instance (Claude Code, Codex, OpenCode, …) running inside a sandbox. Does the actual work; responds in chat. | Inner layer only |
+| **Sandbox** | A multi-harness VM. Hosts **1..N workers**, which can share a workspace. Not an agent — the substrate an agent runs in. | Inner layer only |
+| **Measurement** | Runs the worker over a set of scenarios and judges the outputs into a scorecard with confidence intervals. This is `runCampaign`. | Outer layer |
+
+Two facts that trip people up:
+
+1. **A sandbox is not a worker.** One sandbox can hold ten workers — a driver
+   can coordinate CC + Codex + OpenCode siblings sharing one workspace, or a
+   fleet spread across machines. `runLoop`'s placement encodes exactly this:
+   `{ sibling, sandboxId }` = co-located workers; `{ fleet, fleetId,
+   machineId, sandboxId }` = workers across machines.
+
+2. **"Driver" exists at two layers and means the same *kind* of thing
+   (a decider) at each, but the things it decides differ:**
+   - **Conversation driver** (inner): decides the next *turn* — a persona/user
+     simulating chat, or a planner fanning work to workers.
+   - **Improvement driver** (outer): decides the next *surface* — what system
+     prompt / tool config / code the workers should run.
+
+## The nesting
+
+There are two loops. The outer one improves the thing the inner one runs.
+
+```
+runImprovementLoop                         OUTER loop — improve the agent over time
+│
+├─ DRIVER = ImprovementDriver              proposes a candidate SURFACE
+│           (evolutionary mutator |        (the worker's system prompt / tools / config)
+│            reflective analyst)            — NOT a conversation turn
+│
+└─ for each candidate surface:
+     │
+     runCampaign                           a MEASUREMENT — scores ONE surface
+     │
+     └─ for each scenario × rep:
+          │
+          dispatch(scenario)               THE SEAM — topology-opaque, returns an artifact
+          │
+          └─ runLoop / runMultishot        INNER loop — one conversation
+               ├─ DRIVER  = persona / user / planner    chats with ↓
+               └─ WORKERS = 1..N agent harnesses in 1..M sandboxes
+               │
+               → transcript / artifact
+        judge(artifact) → score
+   → scorecard + CIs
+  gate(winner vs baseline) → PR
+```
+
+### `dispatch` is the topology-opaque seam
+
+`dispatch(scenario) → artifact` is the boundary between the measurement layer
+and the execution layer. The measurement does **not** know or care how the
+artifact was produced. Behind the seam can be:
+
+- one LLM call,
+- one worker (CC) in one sandbox,
+- a conversation driver coordinating 10 workers (CC + Codex + OpenCode)
+  sharing a workspace in one sandbox,
+- a fleet across machines.
+
+All of it is invisible to `runCampaign`. This is why the substrate has no
+opinion about execution topology: the topology lives inside `dispatch`.
+
+### Corrected statements (things that were said backwards)
+
+- The worker is the agent in the sandbox. The driver talks to it. ✓
+- `runCampaign` is a **measurement**, not a worker. It *runs the worker* (via
+  `dispatch`); the worker does not "run the eval".
+- The outer improvement loop has **no single worker** — its driver proposes a
+  *surface*, and each surface is scored by a *measurement* that drives the
+  inner workers.
+
+## The dataset flywheel — why every loop run matters
+
+**Every loop run, regardless of why it ran, feeds the same dataset.** This is
+the through-line that ties measurement and improvement together.
+
+When `runCampaign` runs with a `labeledStore`, each cell captures
+`(scenario, artifact, judgeScore, source)` into the `LabeledScenarioStore`.
+The `source` discriminates *why* the run happened — but the captured tuple is
+identical in shape:
+
+| `captureSource` | The run that produced it |
+|---|---|
+| `'eval-run'` | a plain evaluation campaign |
+| `'production-trace'` | a real user conversation in production |
+| `'red-team'` | an adversarial probe |
+| `'synthetic'` | a generated scenario |
+| `'manual'` | a human-curated example |
+
+That captured corpus **is the GEPA training set.** A basic eval run, a
+production conversation, and an autoresearch loop all deposit the same
+`(input, output, reward)` tuples. The optimization driver later samples from
+that corpus to evolve the surface. So:
+
+> Running *any* loop — even one whose purpose is not optimization — builds the
+> dataset that optimization needs. The flywheel turns whether or not you are
+> currently optimizing.
+
+This is enforced, not aspirational: `runImprovementLoop` **refuses**
+`tracing: 'off'` whenever a driver is wired, precisely because a loop that
+doesn't feed the dataset is a loop that breaks the flywheel.
+
+Temporal-split discipline (train vs holdout, `capturedBefore`) and
+default-off-for-training of `production-trace` are enforced at the
+`LabeledScenarioStore.sample()` boundary so the flywheel cannot contaminate
+the holdout it is judged against. See `src/campaign/labeled-store/`.
+
+## One improvement loop, pluggable drivers
+
+The improvement loop is **driver-agnostic**. `runOptimization` (the loop body)
+and `runImprovementLoop` (the gated-promotion shell) call
+`driver.propose(...)` → measure → `driver.decide(...)`. They do not know which
+strategy is driving. Two strategies conform to the same `ImprovementDriver`
+interface:
+
+```ts
+interface ImprovementDriver<TFindings = unknown> {
+  kind: string
+  propose(args: {
+    currentSurface: MutableSurface
+    history: GenerationRecord[]   // what's been tried + scored
+    findings: TFindings[]         // external signal (e.g. analyst output)
+    populationSize: number
+    generation: number
+    signal: AbortSignal
+  }): Promise<MutableSurface[]>
+  decide?(args: { history: GenerationRecord[] }): { stop: boolean; reason?: string }
+}
+```
+
+| Driver | Strategy | How it proposes | Where it lives |
+|---|---|---|---|
+| `evolutionaryDriver` | Evolutionary (GEPA / AxGEPA) | Mutates the current best surface into N candidates, blind to history beyond the current best. Optimizes against the dataset's rewards. | **agent-eval** (pure: dataset → surface, no sandbox) |
+| `analystDriver` *(planned)* | Reflective | Reads trace findings + generation history, reasons about *why* candidates failed, proposes targeted edits. | **agent-runtime** (runs sandboxes to do research) — implements agent-eval's `ImprovementDriver` |
+
+This resolves the prior duplication where `runImprovementLoop` (evolutionary,
+agent-eval) and `runAnalystLoop` (reflective, agent-runtime) were two parallel
+loops doing "propose change → measure → gate → PR". There is **one loop**;
+the analyst becomes a driver of it. The dependency direction permits this
+cleanly: agent-eval is the leaf and owns the `ImprovementDriver` contract;
+agent-runtime imports agent-eval and implements the contract.
+
+## What "the surface" is — improvement tiers
+
+`MutableSurface` is the thing a driver changes. It has tiers, least → most
+invasive. Today `MutableSurface = string` models tiers 1–2; tiers 3–4 are the
+open design question below.
+
+| Tier | Surface | Driver that changes it | Blast radius |
+|---|---|---|---|
+| 1 | System prompt / prompt-signature addendum | `evolutionaryDriver` (GEPA), `analystDriver` | prompt only |
+| 2 | Tool config / tool signatures | `analystDriver` | which tools, their schemas |
+| 3 | Knowledge (wiki / knowledge graph) | agent-knowledge's knowledge adapter | what the agent *knows* |
+| 4 | Code / scaffolding | autoresearch (reads codebase + traces) → worktree / PR | the implementation itself |
+
+The key distinction Drew drew:
+
+- **Analyst** updates the *signatures* — the prompt and tool surface (tiers
+  1–2). Cheap, reversible, measured directly against the dataset.
+- **Autoresearch** updates the *code* (tier 4). It reads the repository plus
+  the trace findings, opens a worktree, and proposes implementation changes —
+  measured by re-running the inner loop against the changed code.
+
+Both are `ImprovementDriver`s in the abstract (propose a change → measure →
+gate → PR). They differ only in *what* they edit and *how invasive* it is. And
+both consume the **same dataset** the flywheel builds.
+
+## Resolved design decisions
+
+1. **`MutableSurface` widens to span all tiers.** `MutableSurface = string |
+   CodeSurface`. The `string` form is tiers 1–2 (prompt / serialized tool
+   config); `CodeSurface = { kind: 'code'; worktreeRef; baseRef?; summary? }`
+   is tier 4 (an implementation change behind a worktree ref). One loop spans
+   prompt *and* code improvement. `surfaceHash` hashes a string by content and
+   a code surface by its `(worktreeRef, baseRef)` identity (the content lives
+   in git). **Shipped in agent-eval 0.40.1.** The consumer's
+   `dispatchWithSurface` is responsible for checking out a code surface's
+   worktree before running the worker.
+
+2. **`runAnalystLoop` (agent-runtime): analyst becomes a driver; knowledge
+   stays separate.** Extract an `analystDriver` (implements agent-eval's
+   `ImprovementDriver`) for the surface-proposal part, and feed it into
+   `runImprovementLoop`'s gate + PR machinery. `runAnalystLoop`'s other
+   responsibilities — the findings ledger and knowledge-graph updates, which
+   are *not* surface optimization — stay where they are. **Phase 3
+   (agent-runtime); the `ImprovementDriver` contract it implements is already
+   shipped in agent-eval 0.40.1.**
+
+3. **`runLoop` + `runMultishot` converge into one parameterized
+   `runConversationLoop`** with a pluggable backend (`sandbox | router`). The
+   two are the same shape (driver ↔ workers, iterate) differing only in
+   backend and intent; unify them. **Phase 3+ (cross-repo); needs its own
+   design pass — introduces a backend abstraction and couples the two repos'
+   inner loops, so it lands after the `ImprovementDriver` model is proven in
+   product use.**
+
+## Vocabulary quick reference
+
+- **shot** — one conversational turn (driver says X, worker responds Y). Used
+  in `runMultishot`. Never used to mean a whole eval run.
+- **runMultishot** — many shots in one conversation; persona-driver ↔ one
+  router-agent. agent-eval.
+- **runLoop** — driver ↔ workers in sandboxes; topology-agnostic execution.
+  agent-runtime.
+- **runCampaign** — a measurement: a surface scored over N scenarios × M reps.
+  agent-eval. (A "campaign" = a coordinated batch of measurements.)
+- **runOptimization** — the improvement loop body: driver proposes surfaces,
+  each measured by a campaign, top-K promoted per generation. agent-eval.
+- **runImprovementLoop** — `runOptimization` + holdout re-score + release gate
+  + optional PR. agent-eval.
+- **runAnalystLoop** — reflective autoresearch: findings + knowledge updates +
+  improvement proposals. agent-runtime.
+- **ImprovementDriver** — the pluggable strategy that proposes surfaces;
+  `evolutionaryDriver` and (planned) `analystDriver` conform.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.37.0",
+  "version": "0.40.1",
   "description": "Substrate for self-improving agents: traces, verifiable rewards, preferences, GEPA / reflective mutation, auto-research, replay, sequential anytime-valid stats, and release gates.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -99,6 +99,16 @@
       "import": "./dist/matrix/index.js",
       "default": "./dist/matrix/index.js"
     },
+    "./multishot": {
+      "types": "./dist/multishot/index.d.ts",
+      "import": "./dist/multishot/index.js",
+      "default": "./dist/multishot/index.js"
+    },
+    "./campaign": {
+      "types": "./dist/campaign/index.d.ts",
+      "import": "./dist/campaign/index.js",
+      "default": "./dist/campaign/index.js"
+    },
     "./openapi.json": {
       "default": "./dist/openapi.json"
     }
@@ -114,17 +124,6 @@
   "publishConfig": {
     "access": "public"
   },
-  "scripts": {
-    "build": "tsup && pnpm openapi",
-    "dev": "tsup --watch",
-    "prepare": "pnpm build",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "typecheck": "tsc --noEmit",
-    "lint": "biome check src",
-    "format": "biome format --write src",
-    "openapi": "node dist/cli.js openapi --out dist/openapi.json"
-  },
   "dependencies": {
     "@asteasolutions/zod-to-openapi": "^8.5.0",
     "@ax-llm/ax": "^19.0.25",
@@ -138,30 +137,45 @@
     "@tangle-network/sandbox": "^0.2.1"
   },
   "peerDependenciesMeta": {
-    "@tangle-network/agent-runtime": { "optional": true },
-    "@tangle-network/sandbox": { "optional": true }
+    "@tangle-network/agent-runtime": {
+      "optional": true
+    },
+    "@tangle-network/sandbox": {
+      "optional": true
+    }
   },
   "devDependencies": {
     "@biomejs/biome": "^2.4.15",
     "@tangle-network/agent-runtime": "^0.21.0",
     "@tangle-network/sandbox": "^0.2.1",
     "@types/node": "^25.6.0",
+    "husky": "^9.1.7",
+    "lint-staged": "^17.0.5",
     "openapi3-ts": "^4.5.0",
     "tsup": "^8.0.0",
     "typescript": "^5.7.0",
     "vitest": "^3.0.0"
   },
-  "pnpm": {
-    "minimumReleaseAge": 4320,
-    "minimumReleaseAgeExclude": ["@tangle-network/sandbox", "@tangle-network/agent-runtime"],
-    "overrides": {
-      "postcss@<8.5.10": "^8.5.10",
-      "ws@>=8.0.0 <8.20.1": "^8.20.1"
-    }
-  },
   "engines": {
     "node": ">=20"
   },
+  "lint-staged": {
+    "src/**/*.{ts,tsx}": [
+      "biome check --write --no-errors-on-unmatched"
+    ],
+    "tests/**/*.{ts,tsx}": [
+      "biome check --write --no-errors-on-unmatched"
+    ]
+  },
   "license": "MIT",
-  "packageManager": "pnpm@10.22.0"
-}
+  "scripts": {
+    "build": "tsup && pnpm openapi",
+    "dev": "tsup --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit",
+    "lint": "biome check src",
+    "format": "biome format --write src",
+    "openapi": "node dist/cli.js openapi --out dist/openapi.json"
+  }
+}