@tangle-network/agent-eval 0.38.0 → 0.40.2

package/dist/run-campaign-JYJXYHHL.js

@@ -0,0 +1,10 @@
+import {
+  runCampaign
+} from "./chunk-TMXPFWC7.js";
+import "./chunk-WP7SY7AI.js";
+import "./chunk-QYJT52YW.js";
+import "./chunk-PZ5AY32C.js";
+export {
+  runCampaign
+};
+//# sourceMappingURL=run-campaign-JYJXYHHL.js.map

package/dist/run-campaign-JYJXYHHL.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/traces.js

@@ -1,11 +1,9 @@
 import {
-  DEFAULT_REDACTION_RULES,
   DEFAULT_TRACE_ANALYST_BUDGETS,
   FileSystemTraceStore,
   InMemoryTraceStore,
   OTEL_AGENT_EVAL_SCOPE,
   OtlpFileTraceStore,
-  REDACTION_VERSION,
   ReplayCache,
   ReplayCacheMissError,
   SpanNotFoundError,
@@ -30,13 +28,17 @@ import {
   iterateRawCalls,
   otelRunCompleteHook,
   planTraceInsightQuestions,
-  redactString,
-  redactValue,
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystFunctionGroup,
   traceAnalystOnRunComplete
-} from "./chunk-L5UNCDAJ.js";
+} from "./chunk-MAOZCN36.js";
+import {
+  DEFAULT_REDACTION_RULES,
+  REDACTION_VERSION,
+  redactString,
+  redactValue
+} from "./chunk-GGE4NNQT.js";
 import {
   aggregateLlm,
   argHash,

package/dist/wire/index.d.ts

@@ -1,4 +1,4 @@
-import { n as FeedbackTrajectoryStore } from '../feedback-trajectory-iATEAHmc.js';
+import { n as FeedbackTrajectoryStore } from '../feedback-trajectory-Dvy-bt7x.js';
 import { T as TraceStore } from '../store-Db2Bv8Cf.js';
 import { z } from 'zod';
 import { OpenAPIObject } from 'openapi3-ts/oas31';
@@ -7,7 +7,7 @@ import { ServerType } from '@hono/node-server';
 import { Hono } from 'hono';
 import '../control-runtime-BZ_lVLYW.js';
 import '../emitter-DP_cSSiw.js';
-import '../dataset-ueRVTUoY.js';
+import '../dataset-BlwAtYYf.js';
 import '../errors-mje_cKOs.js';
 
 declare const RubricDimensionSchema: z.ZodObject<{

package/docs/design/loop-taxonomy.md

@@ -0,0 +1,233 @@
+# Loop taxonomy: driver, worker, measurement, and the improvement loop
+
+This is the canonical vocabulary for the Tangle agent stack. It exists because
+the same word ("loop", "shot", "worker") was being used at three different
+layers, and the layers were getting conflated. Every role below has exactly
+one meaning. Use these words and nothing else.
+
+Cross-links: [`three-package-architecture.md`](../three-package-architecture.md)
+(who owns what), [`concepts.md`](../concepts.md) (eval mental model),
+[`multi-shot-optimization.md`](../multi-shot-optimization.md) (GEPA),
+[`auto-research-loop-end-to-end.md`](../auto-research-loop-end-to-end.md)
+(analyst / autoresearch).
+
+## The three roles
+
+| Role | Definition | Lives at |
+|---|---|---|
+| **Driver** | The thing that *decides what happens next*. Plans, then decides whether to continue. | Both layers (see below) |
+| **Worker** | An agent harness instance (Claude Code, Codex, OpenCode, …) running inside a sandbox. Does the actual work; responds in chat. | Inner layer only |
+| **Sandbox** | A multi-harness VM. Hosts **1..N workers**, which can share a workspace. Not an agent — the substrate an agent runs in. | Inner layer only |
+| **Measurement** | Runs the worker over a set of scenarios and judges the outputs into a scorecard with confidence intervals. This is `runCampaign`. | Outer layer |
+
+Two facts that trip people up:
+
+1. **A sandbox is not a worker.** One sandbox can hold ten workers — a driver
+   can coordinate CC + Codex + OpenCode siblings sharing one workspace, or a
+   fleet spread across machines. `runLoop`'s placement encodes exactly this:
+   `{ sibling, sandboxId }` = co-located workers; `{ fleet, fleetId,
+   machineId, sandboxId }` = workers across machines.
+
+2. **"Driver" exists at two layers and means the same *kind* of thing
+   (a decider) at each, but the things it decides differ:**
+   - **Conversation driver** (inner): decides the next *turn* — a persona/user
+     simulating chat, or a planner fanning work to workers.
+   - **Improvement driver** (outer): decides the next *surface* — what system
+     prompt / tool config / code the workers should run.
+
+## The nesting
+
+There are two loops. The outer one improves the thing the inner one runs.
+
+```
+runImprovementLoop                         OUTER loop — improve the agent over time
+│
+├─ DRIVER = ImprovementDriver              proposes a candidate SURFACE
+│           (evolutionary mutator |        (the worker's system prompt / tools / config)
+│            reflective analyst)            — NOT a conversation turn
+│
+└─ for each candidate surface:
+     │
+     runCampaign                           a MEASUREMENT — scores ONE surface
+     │
+     └─ for each scenario × rep:
+          │
+          dispatch(scenario)               THE SEAM — topology-opaque, returns an artifact
+          │
+          └─ runLoop / runMultishot        INNER loop — one conversation
+               ├─ DRIVER  = persona / user / planner    chats with ↓
+               └─ WORKERS = 1..N agent harnesses in 1..M sandboxes
+               │
+               → transcript / artifact
+        judge(artifact) → score
+   → scorecard + CIs
+  gate(winner vs baseline) → PR
+```
+
+### `dispatch` is the topology-opaque seam
+
+`dispatch(scenario) → artifact` is the boundary between the measurement layer
+and the execution layer. The measurement does **not** know or care how the
+artifact was produced. Behind the seam can be:
+
+- one LLM call,
+- one worker (CC) in one sandbox,
+- a conversation driver coordinating 10 workers (CC + Codex + OpenCode)
+  sharing a workspace in one sandbox,
+- a fleet across machines.
+
+All of it is invisible to `runCampaign`. This is why the substrate has no
+opinion about execution topology: the topology lives inside `dispatch`.
+
+### Corrected statements (things that were said backwards)
+
+- The worker is the agent in the sandbox. The driver talks to it. ✓
+- `runCampaign` is a **measurement**, not a worker. It *runs the worker* (via
+  `dispatch`); the worker does not "run the eval".
+- The outer improvement loop has **no single worker** — its driver proposes a
+  *surface*, and each surface is scored by a *measurement* that drives the
+  inner workers.
+
+## The dataset flywheel — why every loop run matters
+
+**Every loop run, regardless of why it ran, feeds the same dataset.** This is
+the through-line that ties measurement and improvement together.
+
+When `runCampaign` runs with a `labeledStore`, each cell captures
+`(scenario, artifact, judgeScore, source)` into the `LabeledScenarioStore`.
+The `source` discriminates *why* the run happened — but the captured tuple is
+identical in shape:
+
+| `captureSource` | The run that produced it |
+|---|---|
+| `'eval-run'` | a plain evaluation campaign |
+| `'production-trace'` | a real user conversation in production |
+| `'red-team'` | an adversarial probe |
+| `'synthetic'` | a generated scenario |
+| `'manual'` | a human-curated example |
+
+That captured corpus **is the GEPA training set.** A basic eval run, a
+production conversation, and an autoresearch loop all deposit the same
+`(input, output, reward)` tuples. The optimization driver later samples from
+that corpus to evolve the surface. So:
+
+> Running *any* loop — even one whose purpose is not optimization — builds the
+> dataset that optimization needs. The flywheel turns whether or not you are
+> currently optimizing.
+
+This is enforced, not aspirational: `runImprovementLoop` **refuses**
+`tracing: 'off'` whenever a driver is wired, precisely because a loop that
+doesn't feed the dataset is a loop that breaks the flywheel.
+
+Temporal-split discipline (train vs holdout, `capturedBefore`) and
+default-off-for-training of `production-trace` are enforced at the
+`LabeledScenarioStore.sample()` boundary so the flywheel cannot contaminate
+the holdout it is judged against. See `src/campaign/labeled-store/`.
+
+## One improvement loop, pluggable drivers
+
+The improvement loop is **driver-agnostic**. `runOptimization` (the loop body)
+and `runImprovementLoop` (the gated-promotion shell) call
+`driver.propose(...)` → measure → `driver.decide(...)`. They do not know which
+strategy is driving. Two strategies conform to the same `ImprovementDriver`
+interface:
+
+```ts
+interface ImprovementDriver<TFindings = unknown> {
+  kind: string
+  propose(args: {
+    currentSurface: MutableSurface
+    history: GenerationRecord[]   // what's been tried + scored
+    findings: TFindings[]         // external signal (e.g. analyst output)
+    populationSize: number
+    generation: number
+    signal: AbortSignal
+  }): Promise<MutableSurface[]>
+  decide?(args: { history: GenerationRecord[] }): { stop: boolean; reason?: string }
+}
+```
+
+| Driver | Strategy | How it proposes | Where it lives |
+|---|---|---|---|
+| `evolutionaryDriver` | Evolutionary (GEPA / AxGEPA) | Mutates the current best surface into N candidates, blind to history beyond the current best. Optimizes against the dataset's rewards. | **agent-eval** (pure: dataset → surface, no sandbox) |
+| `analystDriver` *(planned)* | Reflective | Reads trace findings + generation history, reasons about *why* candidates failed, proposes targeted edits. | **agent-runtime** (runs sandboxes to do research) — implements agent-eval's `ImprovementDriver` |
+
+This resolves the prior duplication where `runImprovementLoop` (evolutionary,
+agent-eval) and `runAnalystLoop` (reflective, agent-runtime) were two parallel
+loops doing "propose change → measure → gate → PR". There is **one loop**;
+the analyst becomes a driver of it. The dependency direction permits this
+cleanly: agent-eval is the leaf and owns the `ImprovementDriver` contract;
+agent-runtime imports agent-eval and implements the contract.
+
+## What "the surface" is — improvement tiers
+
+`MutableSurface` is the thing a driver changes. It has tiers, least → most
+invasive. Today `MutableSurface = string` models tiers 1–2; tiers 3–4 are the
+open design question below.
+
+| Tier | Surface | Driver that changes it | Blast radius |
+|---|---|---|---|
+| 1 | System prompt / prompt-signature addendum | `evolutionaryDriver` (GEPA), `analystDriver` | prompt only |
+| 2 | Tool config / tool signatures | `analystDriver` | which tools, their schemas |
+| 3 | Knowledge (wiki / knowledge graph) | agent-knowledge's knowledge adapter | what the agent *knows* |
+| 4 | Code / scaffolding | autoresearch (reads codebase + traces) → worktree / PR | the implementation itself |
+
+The key distinction Drew drew:
+
+- **Analyst** updates the *signatures* — the prompt and tool surface (tiers
+  1–2). Cheap, reversible, measured directly against the dataset.
+- **Autoresearch** updates the *code* (tier 4). It reads the repository plus
+  the trace findings, opens a worktree, and proposes implementation changes —
+  measured by re-running the inner loop against the changed code.
+
+Both are `ImprovementDriver`s in the abstract (propose a change → measure →
+gate → PR). They differ only in *what* they edit and *how invasive* it is. And
+both consume the **same dataset** the flywheel builds.
+
+## Resolved design decisions
+
+1. **`MutableSurface` widens to span all tiers.** `MutableSurface = string |
+   CodeSurface`. The `string` form is tiers 1–2 (prompt / serialized tool
+   config); `CodeSurface = { kind: 'code'; worktreeRef; baseRef?; summary? }`
+   is tier 4 (an implementation change behind a worktree ref). One loop spans
+   prompt *and* code improvement. `surfaceHash` hashes a string by content and
+   a code surface by its `(worktreeRef, baseRef)` identity (the content lives
+   in git). **Shipped in agent-eval 0.40.1.** The consumer's
+   `dispatchWithSurface` is responsible for checking out a code surface's
+   worktree before running the worker.
+
+2. **`runAnalystLoop` (agent-runtime): analyst becomes a driver; knowledge
+   stays separate.** Extract an `analystDriver` (implements agent-eval's
+   `ImprovementDriver`) for the surface-proposal part, and feed it into
+   `runImprovementLoop`'s gate + PR machinery. `runAnalystLoop`'s other
+   responsibilities — the findings ledger and knowledge-graph updates, which
+   are *not* surface optimization — stay where they are. **Phase 3
+   (agent-runtime); the `ImprovementDriver` contract it implements is already
+   shipped in agent-eval 0.40.1.**
+
+3. **`runLoop` + `runMultishot` converge into one parameterized
+   `runConversationLoop`** with a pluggable backend (`sandbox | router`). The
+   two are the same shape (driver ↔ workers, iterate) differing only in
+   backend and intent; unify them. **Phase 3+ (cross-repo); needs its own
+   design pass — introduces a backend abstraction and couples the two repos'
+   inner loops, so it lands after the `ImprovementDriver` model is proven in
+   product use.**
+
+## Vocabulary quick reference
+
+- **shot** — one conversational turn (driver says X, worker responds Y). Used
+  in `runMultishot`. Never used to mean a whole eval run.
+- **runMultishot** — many shots in one conversation; persona-driver ↔ one
+  router-agent. agent-eval.
+- **runLoop** — driver ↔ workers in sandboxes; topology-agnostic execution.
+  agent-runtime.
+- **runCampaign** — a measurement: a surface scored over N scenarios × M reps.
+  agent-eval. (A "campaign" = a coordinated batch of measurements.)
+- **runOptimization** — the improvement loop body: driver proposes surfaces,
+  each measured by a campaign, top-K promoted per generation. agent-eval.
+- **runImprovementLoop** — `runOptimization` + holdout re-score + release gate
+  + optional PR. agent-eval.
+- **runAnalystLoop** — reflective autoresearch: findings + knowledge updates +
+  improvement proposals. agent-runtime.
+- **ImprovementDriver** — the pluggable strategy that proposes surfaces;
+  `evolutionaryDriver` and (planned) `analystDriver` conform.

package/docs/design/self-improvement-engine.md

@@ -0,0 +1,130 @@
+# The self-improvement engine
+
+How the pieces compose into a closed loop that improves an agent over time.
+This builds on [`loop-taxonomy.md`](./loop-taxonomy.md) (the role vocabulary)
+— read that first. Here we describe the *engine*: the phases, the data flow,
+and where each existing primitive plugs in.
+
+## The closed loop, by phase
+
+```
+PHASE 1 — RUN
+  driver ↔ workers (sandbox) over scenarios
+  → traces emitted → TraceStore + LabeledScenarioStore (the dataset)
+  Every run feeds the dataset regardless of why it ran (see the flywheel
+  section in loop-taxonomy.md). This is the only source of improvement signal.
+
+PHASE 2 — ANALYZE                                  ← the research report is born here
+  trace analysts run over the accumulated traces
+  (today: runAnalystLoop steps 2–4 in agent-runtime)
+    - run the analyst registry over traces → findings
+    - persist findings to the ledger
+    - diff the new findings vs the baseline → research report
+  Output: a research report = { findings, diff } grounded in real traces.
+
+PHASE 3 — PROPOSE
+  ImprovementDriver.propose(input) → MutableSurface[]
+  input carries:
+    - currentSurface      the current best surface (prompt string or CodeSurface)
+    - history             prior generations + their scores
+    - report              the Phase-2 research report (findings + diff)
+    - traces              all traces (read access) — "all the data"
+    - dataset             the LabeledScenarioStore handle
+    - populationSize      BREADTH: how many candidate surfaces to return
+    - maxImprovementShots DEPTH: how many runLoop iterations each candidate
+                          generation may take (1..MAX_IMPROVEMENT_SHOTS)
+  For the code-tier (autoresearch) driver, propose() runs a FULL sandbox
+  runLoop: a driver↔worker(s) loop that reads report+traces+codebase and
+  produces the improvement as commits in ONE worktree per candidate.
+  Output: CodeSurface{ worktreeRef }[] (or string[] for prompt-tier).
+
+PHASE 4 — MEASURE
+  each candidate → runCampaign on the holdout set
+  (checks out the candidate's worktree, runs the worker against the changed
+   code/prompt, judges, scores). The measurement is driver-agnostic.
+
+PHASE 5 — GATE + PROMOTE
+  defaultProductionGate(winner vs baseline on holdout) → ship | hold | …
+  on ship → open a PR from the winning worktree (one worktree = one PR).
+
+  ↺ loop back to PHASE 1 with the promoted surface as the new baseline.
+```
+
+The improvement loop body (`runOptimization`) owns Phases 3–4; the gated
+shell (`runImprovementLoop`) adds Phase 5. Phases 1–2 are upstream — the
+run that produces traces, and the analysts that turn traces into a report.
+
+## `propose()` — the plan step, recursively agentic
+
+`propose()` does NOT run the worker and does NOT measure. It returns N
+candidate surfaces to measure next. *How* it produces them is per-driver and
+spans a cost spectrum:
+
+| Driver | `propose()` mechanism | Sandbox? | Output |
+|---|---|---|---|
+| `evolutionaryDriver` | mutate current surface text into N variants | no | `string[]` |
+| `analystDriver` (reflective) | LLM reads the report → drafts edits | LLM call | `string[]` / `CodeSurface[]` |
+| `autoresearchDriver` (code-tier) | **full sandbox runLoop** (≤ `maxImprovementShots`) reads report+traces+codebase → commits in one worktree | **yes** | `CodeSurface[]` |
+
+The recursion: generating *one* candidate (autoresearch `propose`) is itself a
+driver↔worker-in-a-sandbox loop, nested inside the *measurement* of that
+candidate (Phase 4), nested inside the improvement loop. "A loop whose step
+contains a loop."
+
+Two knobs, not one:
+- **`populationSize`** — breadth: how many candidates `propose()` returns.
+- **`maxImprovementShots`** — depth: how many runLoop iterations the
+  generating agent gets per candidate (N=1 → single-shot; N>1 → it can
+  iterate on its own change before handing it back to be measured).
+
+## Package boundaries (respecting the leaf direction)
+
+agent-eval is the leaf (imports nothing upstream). agent-runtime imports it.
+So:
+
+| Piece | Package | Why |
+|---|---|---|
+| `ImprovementDriver` contract | agent-eval | the shared interface; everyone implements it |
+| widened `propose()` input (report/traces/dataset) | agent-eval | part of the contract |
+| `evolutionaryDriver` | agent-eval | pure: dataset → surface, no sandbox |
+| **VCS-pluggable worktree adapter** | agent-eval | pure git/FS, no sandbox; produces `CodeSurface` |
+| `runOptimization` / `runImprovementLoop` | agent-eval | driver-agnostic loop body + gated shell |
+| `defaultProductionGate` | agent-eval | measurement-side safety |
+| **`autoresearchDriver`** (sandbox-spawning `propose`) | agent-runtime | needs the sandbox SDK + `runLoop` |
+| `analystDriver` (wraps the improvement adapter) | agent-runtime | depends on `runAnalystLoop` machinery |
+| trace analysts / `runAnalystLoop` (Phase 2) | agent-runtime | runs agents to analyze |
+
+## The worktree adapter (VCS-pluggable)
+
+One improvement = one worktree, PR-like (multiple commits allowed). The
+adapter abstracts the VCS so the driver code is VCS-agnostic:
+
+```ts
+interface WorktreeAdapter {
+  create(opts: { baseRef: string; label: string }): Promise<Worktree>
+  // ... agent commits into worktree.path ...
+  finalize(wt: Worktree, summary: string): Promise<CodeSurface>  // → { kind:'code', worktreeRef, baseRef, summary }
+  discard(wt: Worktree): Promise<void>
+}
+```
+
+- **git** impl ships first (`git worktree add` / branch / commit).
+- **jj** ([jj-vcs](https://github.com/jj-vcs/jj)) is a candidate second impl —
+  not built now; the interface exists so it can slot in without touching
+  driver code.
+
+The measurement (Phase 4) consumes a `CodeSurface` by checking out
+`worktreeRef` before running the worker; on promotion (Phase 5) the worktree
+becomes the PR branch.
+
+## Build sequence
+
+1. **agent-eval 0.40.2**: widen `propose()` input (additive optional
+   `report` / `traces` / `dataset` / `maxImprovementShots`); add the
+   VCS-pluggable worktree adapter with a git impl; multi-sink trace fan-out
+   helper.
+2. **agent-runtime 0.25.0**: `analystDriver` (wraps the improvement adapter,
+   fed the Phase-2 report); `autoresearchDriver` (sandbox runLoop `propose`);
+   default-on multi-sink tracing in `handleChatTurn`.
+3. Wire one consumer end-to-end (Phase 4 of the broader rollout), prove it,
+   then fan out.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.38.0",
+  "version": "0.40.2",
   "description": "Substrate for self-improving agents: traces, verifiable rewards, preferences, GEPA / reflective mutation, auto-research, replay, sequential anytime-valid stats, and release gates.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -104,6 +104,11 @@
       "import": "./dist/multishot/index.js",
       "default": "./dist/multishot/index.js"
     },
+    "./campaign": {
+      "types": "./dist/campaign/index.d.ts",
+      "import": "./dist/campaign/index.js",
+      "default": "./dist/campaign/index.js"
+    },
     "./openapi.json": {
       "default": "./dist/openapi.json"
     }
@@ -119,17 +124,6 @@
   "publishConfig": {
     "access": "public"
   },
-  "scripts": {
-    "build": "tsup && pnpm openapi",
-    "dev": "tsup --watch",
-    "prepare": "pnpm build",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "typecheck": "tsc --noEmit",
-    "lint": "biome check src",
-    "format": "biome format --write src",
-    "openapi": "node dist/cli.js openapi --out dist/openapi.json"
-  },
   "dependencies": {
     "@asteasolutions/zod-to-openapi": "^8.5.0",
     "@ax-llm/ax": "^19.0.25",
@@ -143,30 +137,45 @@
     "@tangle-network/sandbox": "^0.2.1"
   },
   "peerDependenciesMeta": {
-    "@tangle-network/agent-runtime": { "optional": true },
-    "@tangle-network/sandbox": { "optional": true }
+    "@tangle-network/agent-runtime": {
+      "optional": true
+    },
+    "@tangle-network/sandbox": {
+      "optional": true
+    }
   },
   "devDependencies": {
     "@biomejs/biome": "^2.4.15",
     "@tangle-network/agent-runtime": "^0.21.0",
     "@tangle-network/sandbox": "^0.2.1",
     "@types/node": "^25.6.0",
+    "husky": "^9.1.7",
+    "lint-staged": "^17.0.5",
     "openapi3-ts": "^4.5.0",
     "tsup": "^8.0.0",
     "typescript": "^5.7.0",
     "vitest": "^3.0.0"
   },
-  "pnpm": {
-    "minimumReleaseAge": 4320,
-    "minimumReleaseAgeExclude": ["@tangle-network/sandbox", "@tangle-network/agent-runtime"],
-    "overrides": {
-      "postcss@<8.5.10": "^8.5.10",
-      "ws@>=8.0.0 <8.20.1": "^8.20.1"
-    }
-  },
   "engines": {
     "node": ">=20"
   },
+  "lint-staged": {
+    "src/**/*.{ts,tsx}": [
+      "biome check --write --no-errors-on-unmatched"
+    ],
+    "tests/**/*.{ts,tsx}": [
+      "biome check --write --no-errors-on-unmatched"
+    ]
+  },
   "license": "MIT",
-  "packageManager": "pnpm@10.22.0"
-}
+  "scripts": {
+    "build": "tsup && pnpm openapi",
+    "dev": "tsup --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit",
+    "lint": "biome check src",
+    "format": "biome format --write src",
+    "openapi": "node dist/cli.js openapi --out dist/openapi.json"
+  }
+}