@tangle-network/agent-eval 0.43.1 → 0.44.0

package/dist/{rubric-predictive-validity-ByZEC3BX.d.ts → rubric-predictive-validity-CJ08tGwq.d.ts}

@@ -1,5 +1,5 @@
 import { R as RunRecord } from './run-record-BGY6bHRh.js';
-import { O as OutcomeStore } from './outcome-store-D6KWmYvj.js';
+import { O as OutcomeStore } from './outcome-store-BxJ3DQKJ.js';
 
 /**
  * Rubric predictive validity — does our eval rubric predict deployment

package/dist/run-improvement-loop-CbilHQAb.d.ts

@@ -0,0 +1,401 @@
+import { S as Scenario, d as CampaignResult, j as GateResult, o as Mutator, I as ImprovementDriver, G as Gate, D as DispatchFn, J as JudgeConfig, L as LabeledScenarioStore, e as CampaignTraceWriter, M as MutableSurface, l as GenerationRecord } from './types-DToGONFA.js';
+import { L as LlmClientOptions } from './llm-client-BXVRUZyX.js';
+import { RunRecord } from '@tangle-network/agent-runtime';
+import { R as RedTeamCase } from './red-team-30II1T4o.js';
+
+/**
+ * @experimental
+ *
+ * `openAutoPr` — thin shell-out helper for the `runImprovementLoop` preset's
+ * `autoOnPromote: 'pr'` mode. Substitutes for the per-product PR-opening
+ * code consumers duplicated 4 times. The PR body includes the campaign's
+ * manifest hash, gate verdict, and scorecard summary so reviewers can see
+ * exactly what was promoted + why.
+ *
+ * NOT a deploy mechanism — this only OPENS a PR. The human reviews + merges.
+ * The Shape B (`autoOnPromote: 'config'`) live-runtime-mutation path is
+ * deferred to Pass B with the full shadow / canary / rollback stack.
+ */
+
+interface OpenAutoPrOptions<TArtifact, TScenario extends Scenario> {
+    /** Campaign result to attach to the PR. */
+    result: CampaignResult<TArtifact, TScenario>;
+    /** Gate verdict explaining the promotion. Substrate refuses to open a PR
+     *  when `gate.decision !== 'ship'` — fails loud. */
+    gate: GateResult;
+    /** Promoted surface diff — typically the new system prompt addendum or
+     *  full profile diff. Substrate writes it as the PR body. */
+    promotedDiff: string;
+    /** GH owner/repo target (e.g., `tangle-network/gtm-agent`). */
+    ghOwner: string;
+    ghRepo: string;
+    /** Branch name for the PR. Default `auto/<manifestHash[:12]>`. */
+    branch?: string;
+    /** PR title. Default includes manifest hash. */
+    title?: string;
+    /** Whether to actually open the PR or just dry-run. Default reads
+     *  `GH_AUTO_PR_TOKEN` env — present = open, absent = dry-run. */
+    dryRun?: boolean;
+    /** Test seam — substitute `gh pr create` invocation. */
+    ghExec?: (args: string[]) => {
+        stdout: string;
+        stderr: string;
+        status: number;
+    };
+}
+interface OpenAutoPrResult {
+    opened: boolean;
+    prUrl?: string;
+    dryRun: boolean;
+    reason: string;
+}
+declare function openAutoPr<TArtifact, TScenario extends Scenario>(options: OpenAutoPrOptions<TArtifact, TScenario>): OpenAutoPrResult;
+
+/**
+ * @experimental
+ *
+ * `evolutionaryDriver` — adapts a stateless `Mutator` (population mutation:
+ * GEPA / AxGEPA / reflective-mutation) into an `ImprovementDriver`. This is
+ * the evolutionary strategy: each generation, mutate the current best surface
+ * into N candidates, measure, select. No generation memory beyond the current
+ * surface; the loop body handles ranking + promotion.
+ *
+ * The reflective alternative is agent-runtime's `improvementDriver` with a
+ * `reflectiveGenerator` / `agenticGenerator`: it reasons over the report +
+ * trace findings to propose targeted edits rather than blind mutations. Both
+ * conform to `ImprovementDriver`; the improvement loop is identical regardless
+ * of which drives it.
+ */
+
+interface EvolutionaryDriverOptions<TFindings = unknown> {
+    mutator: Mutator<TFindings>;
+    /** External findings fed to the mutator each generation. Default: []. */
+    findings?: TFindings[];
+}
+declare function evolutionaryDriver<TFindings = unknown>(opts: EvolutionaryDriverOptions<TFindings>): ImprovementDriver<TFindings>;
+
+/**
+ * @experimental
+ *
+ * `gepaDriver` — a reflective `ImprovementDriver` for prompt-tier surfaces.
+ * Each generation it reflects on the prior best candidate's per-scenario
+ * scores + weakest dimensions (the `GenerationCandidate` evidence from
+ * `runOptimization`), asks an LLM to propose targeted rewrites of the current
+ * surface, and returns them as the next population.
+ *
+ * This is the substrate's best-in-class prompt optimizer: surface-agnostic, so
+ * ANY string surface in ANY consumer opts in by selecting it — system prompts,
+ * prompt addenda, judge/reviewer prompts, even a driver's own reflection
+ * prompt. It reuses the generic reflection primitive (`buildReflectionPrompt` /
+ * `parseReflectionResponse`) and the router client; it has NO dependency on the
+ * legacy `runMultiShotOptimization` / `prompt-evolution` orchestration.
+ *
+ * It earns its keep where there is real per-instance signal (which the
+ * dimensional + per-scenario evidence + the `LabeledScenarioStore` flywheel
+ * now provide). For thin-signal surfaces it degrades to plain reflection — so
+ * it is a SELECTABLE driver, never a forced default. On generation 0 (no
+ * history) it reflects on the current surface against the mutation primitives
+ * alone.
+ */
+
+interface GepaDriverOptions {
+    /** Router transport (apiKey/baseUrl). */
+    llm: LlmClientOptions;
+    /** Model that performs the reflection. */
+    model: string;
+    /** What is being optimized — appears in the reflection prompt for orientation. */
+    target: string;
+    /** Surface-specific mutation levers offered to the model. */
+    mutationPrimitives?: string[];
+    /** Top/bottom scenarios surfaced as evidence each generation. Default 3. */
+    evidenceK?: number;
+    /** Reflection sampling temperature. Default 0.7. */
+    temperature?: number;
+    /** Reflection max tokens. Default 6000. */
+    maxTokens?: number;
+}
+declare function gepaDriver(opts: GepaDriverOptions): ImprovementDriver;
+
+/**
+ * @experimental
+ *
+ * Compose multiple `Gate` implementations — every gate must pass for the
+ * composite to ship. Closes the alignment reviewer's "default-only
+ * heldOutGate + costGate would happily promote a reward-hacked prompt"
+ * concern by making safety gates first-class composable defaults.
+ */
+
+/** Compose gates — all must `ship` for the composite to `ship`. First
+ *  non-ship verdict short-circuits the composite verdict, but ALL gates run
+ *  (so the result records every gate's reason — useful for diagnostics). */
+declare function composeGate<TArtifact = unknown, TScenario extends Scenario = Scenario>(...gates: Array<Gate<TArtifact, TScenario>>): Gate<TArtifact, TScenario>;
+
+/**
+ * @experimental
+ *
+ * `defaultProductionGate` — composes the substrate's existing safety
+ * primitives (red-team / reward-hacking / canary / heldout) into a single
+ * Gate.decide shape. Closes the alignment + Anthropic-SI reviewers' "safety
+ * primitives are off the critical path" blocker.
+ *
+ * The composition is opinionated — when consumers wire `runImprovementLoop`,
+ * THIS gate is the default. Consumers can still pass a custom gate to
+ * override; the recommended pattern is to compose THIS gate with whatever
+ * extra domain-specific gates they need (`composeGate(defaultProductionGate(...), customGate)`).
+ */
+
+interface DefaultProductionGateOptions {
+    /** Required: scenarios held out from training; substrate compares
+     *  candidate-on-holdout vs baseline-on-holdout. */
+    holdoutScenarios: Scenario[];
+    /** Minimum mean-composite improvement required to ship. Default 0.5. */
+    deltaThreshold?: number;
+    /** Total $ budget for ALL cells in this campaign — including baseline + candidate.
+     *  Composite verdict refuses to ship when spend exceeded budget. */
+    budgetUsd?: number;
+    /** Red-team cases to probe candidate outputs against. When omitted the
+     *  substrate uses `DEFAULT_RED_TEAM_CORPUS`. Provide a domain-specific
+     *  battery for tighter coverage. */
+    redTeamBattery?: RedTeamCase[];
+    /** Run records (oldest-first) needed for the reward-hacking detector.
+     *  Substrate populates from prior production-loop generations. */
+    recentRuns?: RunRecord[];
+    /** When true, the gate refuses to ship if the reward-hacking detector
+     *  fires at the `gaming` severity. Default true. */
+    blockOnRewardHackingGaming?: boolean;
+}
+declare function defaultProductionGate<TArtifact, TScenario extends Scenario>(options: DefaultProductionGateOptions): Gate<TArtifact, TScenario>;
+
+/**
+ * @experimental
+ *
+ * Thin Gate adapter — exposes delta-threshold-on-holdout as a composable
+ * `Gate`. Use when you want held-out as one of N composed gates instead of
+ * the full `defaultProductionGate` stack.
+ */
+
+interface HeldOutGateOptions<TScenario extends Scenario = Scenario> {
+    scenarios: TScenario[];
+    deltaThreshold?: number;
+}
+declare function heldOutGate<TArtifact, TScenario extends Scenario>(options: HeldOutGateOptions<TScenario>): Gate<TArtifact, TScenario>;
+
+/**
+ * @experimental
+ *
+ * `CampaignStorage` — the filesystem seam `runCampaign` writes through
+ * (run/cell dirs, the resumability cache, per-cell artifacts, trace spans).
+ *
+ * The default (`fsCampaignStorage`) is the Node filesystem — identical
+ * behavior to the inline `node:fs` calls it replaces, so existing CLI
+ * consumers are unaffected. `inMemoryCampaignStorage` keeps everything in a
+ * `Map`, so the substrate runs in environments WITHOUT a filesystem
+ * (Cloudflare Workers, Deno Deploy, other edge runtimes) — the campaign
+ * still produces its `CampaignResult` (cells + aggregates) in memory;
+ * artifacts/traces simply aren't persisted to disk.
+ *
+ * Paths are opaque keys to the in-memory adapter — it does not parse them,
+ * so the same `join(...)`-built paths work unchanged across both adapters.
+ */
+interface CampaignStorage {
+    /** Ensure a directory exists (recursive). No-op for in-memory. */
+    ensureDir(dir: string): void;
+    /** Does this path exist (as a written file or an ensured dir)? */
+    exists(path: string): boolean;
+    /** Read a UTF-8 file; `undefined` when missing or unreadable. */
+    read(path: string): string | undefined;
+    /** Write a file (string or bytes). Parent dir is assumed ensured. */
+    write(path: string, content: string | Uint8Array): void;
+}
+/** Node-filesystem storage — the default. Lazily requires `node:fs` so the
+ *  module imports cleanly in non-Node runtimes (where the caller passes
+ *  `inMemoryCampaignStorage` instead and never constructs this). */
+declare function fsCampaignStorage(): CampaignStorage;
+/** In-memory storage for filesystem-less runtimes. Artifacts + trace spans
+ *  live in a `Map` for the duration of the run; the `CampaignResult` is
+ *  fully populated, but nothing is persisted to disk. */
+declare function inMemoryCampaignStorage(): CampaignStorage;
+
+/**
+ * @experimental
+ *
+ * `runCampaign` — Pass A substrate primitive. ONE function that orchestrates
+ * scenarios → dispatch → artifacts → judges → aggregates, with full
+ * reproducibility (seed + manifest hash), cell-level resumability, bootstrap
+ * CIs, and the `LabeledScenarioStore` capture flywheel.
+ *
+ * Improvement loops (optimizer / gate / autoOnPromote) ride on top of this
+ * primitive but live in `presets/run-improvement-loop.ts`. This file keeps
+ * the core orchestrator minimal — Phase 1 of the Pass A track.
+ */
+
+interface RunCampaignOptions<TScenario extends Scenario, TArtifact> {
+    scenarios: TScenario[];
+    dispatch: DispatchFn<TScenario, TArtifact>;
+    judges?: JudgeConfig<TArtifact, TScenario>[];
+    /** Required for reproducibility. Default 42. */
+    seed?: number;
+    /** Per-scenario replicates for CI bands. Default 1; raise to 5+ for
+     *  bootstrap-tight intervals on critical eval. */
+    reps?: number;
+    /** When true (default), completed cells are cached by
+     *  (manifestHash, scenarioId, rep, generation). Re-runs skip cached cells. */
+    resumable?: boolean;
+    /** Optional store — when present, every artifact + judge score is captured
+     *  with the configured `captureSource`. Capture is default ON; pass `'off'`
+     *  to disable. */
+    labeledStore?: LabeledScenarioStore | 'off';
+    captureSource?: 'production-trace' | 'eval-run' | 'manual' | 'red-team' | 'synthetic';
+    captureSourceVersionHash?: string;
+    /** Wall-clock cost cap across all cells. Cells beyond ceiling are skipped. */
+    costCeiling?: number;
+    /** Max concurrent cells. Default 2. */
+    maxConcurrency?: number;
+    /** Required: where artifacts + traces land. */
+    runDir: string;
+    /** Tracing posture. Default is the substrate's `FileSystemTraceStore` rooted
+     *  at `<runDir>/traces/`. `'off'` disables capture entirely — substrate
+     *  refuses this when the caller wires `autoOnPromote !== 'none'`. */
+    tracing?: 'on' | 'off';
+    /** Test seam — override the wall clock for deterministic tests. */
+    now?: () => Date;
+    /** Test seam — override per-cell trace writer factory. */
+    buildTraceWriter?: (cellId: string, dir: string) => CampaignTraceWriter;
+    /** Storage backend for run/cell dirs, the resumability cache, artifacts,
+     *  and trace spans. Default: the Node filesystem (`fsCampaignStorage`).
+     *  Pass `inMemoryCampaignStorage()` to run in a filesystem-less runtime
+     *  (Cloudflare Workers, Deno, edge) — the `CampaignResult` is still
+     *  produced; artifacts/traces just aren't persisted to disk. */
+    storage?: CampaignStorage;
+}
+declare function runCampaign<TScenario extends Scenario, TArtifact>(opts: RunCampaignOptions<TScenario, TArtifact>): Promise<CampaignResult<TArtifact, TScenario>>;
+
+/**
+ * @experimental
+ *
+ * `runEval` — the simplest preset over `runCampaign`. No optimizer, no
+ * gate, no auto-PR. Just: run scenarios through dispatch, score with
+ * judges, return CampaignResult.
+ *
+ * The 80% case for consumers who want a scorecard, not an improvement loop.
+ */
+
+interface RunEvalOptions<TScenario extends Scenario, TArtifact> extends Omit<RunCampaignOptions<TScenario, TArtifact>, 'runDir'> {
+    runDir: string;
+}
+declare function runEval<TScenario extends Scenario, TArtifact>(opts: RunEvalOptions<TScenario, TArtifact>): Promise<CampaignResult<TArtifact, TScenario>>;
+
+/**
+ * @experimental
+ *
+ * `runOptimization` — the improvement loop body. Runs N generations: the
+ * `ImprovementDriver` proposes K candidate surfaces per generation, each
+ * candidate runs a campaign (the measurement), top-scoring promote to the
+ * next generation. Driver-agnostic — the same loop runs an evolutionary
+ * population mutator (`evolutionaryDriver`) or agent-runtime's
+ * `improvementDriver` (reflective / agentic generators); they differ only in
+ * how `propose()` picks candidates.
+ *
+ * This is `runLoop`'s shape (plan → measure → decide) specialized to surface
+ * improvement: `driver.propose` = plan, `runCampaign` = the measurement (which
+ * runs the worker behind `dispatch`), the mean-composite ranking = the
+ * validator, `driver.decide` = the stop check.
+ *
+ * The gated-promotion shell (`runImprovementLoop`) wraps this with a holdout
+ * re-score + release gate + optional PR.
+ */
+
+interface RunOptimizationOptions<TScenario extends Scenario, TArtifact> extends Omit<RunCampaignOptions<TScenario, TArtifact>, 'dispatch'> {
+    /** Initial mutable surface (typically system prompt or addendum). */
+    baselineSurface: MutableSurface;
+    /** Dispatcher that takes the CURRENT surface + scenario → artifact. */
+    dispatchWithSurface: (surface: MutableSurface, scenario: TScenario, ctx: Parameters<RunCampaignOptions<TScenario, TArtifact>['dispatch']>[1]) => Promise<TArtifact>;
+    /** The improvement strategy. Wrap a population `Mutator` via
+     *  `evolutionaryDriver({ mutator })`, or pass agent-runtime's
+     *  `improvementDriver` (reflective / agentic generators). */
+    driver: ImprovementDriver;
+    populationSize: number;
+    maxGenerations: number;
+    /** How many top-scoring candidates carry to the next generation. Default 2. */
+    promoteTopK?: number;
+    /** DEPTH knob forwarded to the driver's `propose()` — max iterations the
+     *  agentic generator may take per candidate. */
+    maxImprovementShots?: number;
+    /** Phase-2 research report forwarded to `propose()` (analyst findings +
+     *  diff). Opaque here; the driver types it. */
+    report?: unknown;
+}
+interface RunOptimizationResult<TArtifact, TScenario extends Scenario> {
+    generations: Array<{
+        record: GenerationRecord;
+        surfaces: Array<{
+            surfaceHash: string;
+            surface: MutableSurface;
+            campaign: CampaignResult<TArtifact, TScenario>;
+        }>;
+    }>;
+    winnerSurface: MutableSurface;
+    winnerSurfaceHash: string;
+    baselineCampaign: CampaignResult<TArtifact, TScenario>;
+}
+declare function runOptimization<TScenario extends Scenario, TArtifact>(opts: RunOptimizationOptions<TScenario, TArtifact>): Promise<RunOptimizationResult<TArtifact, TScenario>>;
+declare function surfaceHash(surface: MutableSurface): string;
+
+/**
+ * @experimental
+ *
+ * `runImprovementLoop` — the gated-promotion shell around the improvement
+ * loop body (`runOptimization`). Drives candidate surfaces via the
+ * `ImprovementDriver`, re-scores the winner against the baseline on a
+ * holdout set, runs the release gate, and optionally opens a PR.
+ *
+ * Role vocabulary (see docs/design/loop-taxonomy.md):
+ *   - DRIVER     = the `ImprovementDriver` (evolutionary GEPA mutator OR
+ *                  reflective analyst). Proposes candidate SURFACES — the
+ *                  worker's system prompt / tool config — NOT conversation
+ *                  turns.
+ *   - MEASUREMENT= `runCampaign`. Scores one surface by running the worker
+ *                  (via `dispatch`) over scenarios and judging the output.
+ *   - WORKER     = the agent harness in the sandbox, invoked behind the
+ *                  topology-opaque `dispatch` seam — never referenced here.
+ *
+ * Distinct from `runLoop` in `@tangle-network/agent-runtime`, which is the
+ * INNER conversation loop (driver↔workers in a sandbox). `runImprovementLoop`
+ * is the OUTER loop: it improves the surface that those workers run.
+ *
+ * Hard-refuses unsafe configurations:
+ *   - `tracing: 'off'` when a driver is wired (improvement is unattributable)
+ *   - `autoOnPromote: 'config'` — DEFERRED to Pass B; v0.40 only ships
+ *     `'pr'` and `'none'`.
+ */
+
+interface RunImprovementLoopOptions<TScenario extends Scenario, TArtifact> extends RunOptimizationOptions<TScenario, TArtifact> {
+    /** Holdout scenarios kept OUT of the training optimization pool — used
+     *  ONLY to score baseline vs winner for the gate. */
+    holdoutScenarios: TScenario[];
+    /** Promotion gate. Substrate strongly recommends `defaultProductionGate`
+     *  for production wiring (composes red-team / reward-hacking / canary /
+     *  heldout). */
+    gate: Gate<TArtifact, TScenario>;
+    /** What to do when the gate ships:
+     *   - `'pr'`: open a PR via `openAutoPr`
+     *   - `'none'`: just report — caller decides what to do with the winner
+     *  v0.40 does NOT support `'config'` (live-runtime self-mutation) —
+     *  deferred to Pass B behind safety stack. */
+    autoOnPromote: 'pr' | 'none';
+    /** GH owner / repo for the auto-PR. Required when autoOnPromote === 'pr'. */
+    ghOwner?: string;
+    ghRepo?: string;
+    /** Optional render override — substrate writes a diff-shaped surface; pass
+     *  a function to format the promoted surface differently. */
+    renderPromotedDiff?: (winnerSurface: MutableSurface, baselineSurface: MutableSurface) => string;
+}
+interface RunImprovementLoopResult<TArtifact, TScenario extends Scenario> extends RunOptimizationResult<TArtifact, TScenario> {
+    baselineOnHoldout: CampaignResult<TArtifact, TScenario>;
+    winnerOnHoldout: CampaignResult<TArtifact, TScenario>;
+    gateResult: Awaited<ReturnType<Gate<TArtifact, TScenario>['decide']>>;
+    prResult?: ReturnType<typeof openAutoPr>;
+}
+declare function runImprovementLoop<TScenario extends Scenario, TArtifact>(opts: RunImprovementLoopOptions<TScenario, TArtifact>): Promise<RunImprovementLoopResult<TArtifact, TScenario>>;
+
+export { type CampaignStorage as C, type DefaultProductionGateOptions as D, type EvolutionaryDriverOptions as E, type GepaDriverOptions as G, type HeldOutGateOptions as H, type OpenAutoPrOptions as O, type RunCampaignOptions as R, type RunEvalOptions as a, type RunImprovementLoopOptions as b, type RunImprovementLoopResult as c, composeGate as d, defaultProductionGate as e, evolutionaryDriver as f, fsCampaignStorage as g, gepaDriver as h, heldOutGate as i, inMemoryCampaignStorage as j, runEval as k, runImprovementLoop as l, type OpenAutoPrResult as m, type RunOptimizationOptions as n, type RunOptimizationResult as o, openAutoPr as p, runOptimization as q, runCampaign as r, surfaceHash as s };

package/dist/{types-BLbRTxoc.d.ts → types-DToGONFA.d.ts}

@@ -364,4 +364,4 @@ interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scena
     scenarios: Array<Pick<TScenario, 'id' | 'kind'>>;
 }
 
-export type { CampaignResult as C, DispatchFn as D, GateResult as G, ImprovementDriver as I, JudgeConfig as J, LabeledScenarioStore as L, Mutator as M, OptimizerConfig as O, ProposeContext as P, RedactionStatus as R, Scenario as S, TraceSpan as T, Gate as a, LabeledScenarioWrite as b, LabeledScenarioSampleArgs as c, LabeledScenarioRecord as d, CampaignTraceWriter as e, MutableSurface as f, GenerationRecord as g, CodeSurface as h, CampaignAggregates as i, CampaignArtifactWriter as j, CampaignCellResult as k, CampaignCostMeter as l, DispatchContext as m, GateContext as n, GateDecision as o, GenerationCandidate as p, JudgeAggregate as q, JudgeDimension as r, JudgeScore as s, LabeledScenarioSource as t, ScenarioAggregate as u, SessionScript as v };
+export type { CampaignAggregates as C, DispatchFn as D, Gate as G, ImprovementDriver as I, JudgeConfig as J, LabeledScenarioStore as L, MutableSurface as M, OptimizerConfig as O, ProposeContext as P, RedactionStatus as R, Scenario as S, TraceSpan as T, CampaignArtifactWriter as a, CampaignCellResult as b, CampaignCostMeter as c, CampaignResult as d, CampaignTraceWriter as e, CodeSurface as f, DispatchContext as g, GateContext as h, GateDecision as i, GateResult as j, GenerationCandidate as k, GenerationRecord as l, JudgeDimension as m, JudgeScore as n, Mutator as o, SessionScript as p, LabeledScenarioWrite as q, LabeledScenarioSampleArgs as r, LabeledScenarioRecord as s, JudgeAggregate as t, LabeledScenarioSource as u, ScenarioAggregate as v };

package/docs/design/external-agent-wedge.md

@@ -0,0 +1,89 @@
+# External-Agent Wedge — plug-in self-improvement for any agent
+
+**Status:** proposed · **Owner:** Drew · **Tracking:** ops-board #507 (epic) · **Updated:** 2026-05-26
+
+> One sentence: **expose the self-improvement engine our own fleet runs on as a drop-in library any agent builder can install — land free, bill the hosted intelligence.**
+
+This doc locks direction so the team executes without thrash. It is deliberately short. If a question isn't answered here, the default is "do the cheapest thing that keeps the LAND tier free and the EXPAND tier billable."
+
+---
+
+## Why now
+
+Repeated signal from agent founders (latest: a high-quality GTM/social-marketing agent, sharp CTO): **no evals, no observability, hand-built, no way for the agent to improve itself.** This is the median serious agent team. They don't need another trace viewer — they need their agent to *get better on its own* and proof that it did.
+
+We already built that engine and hardened it the hard way: **6 of our own agents** (tax, legal, creative, gtm, agent-builder, physim) now run the *same* `runCampaign` / `runImprovementLoop` / `gepaDriver` / gate surface. It is `@tangle-network/agent-eval` — already on npm, runtime-agnostic, dogfooded in production. Exposing it externally is distribution of an asset we already own, not a new product to invent.
+
+## Positioning — self-improvement first, not observability
+
+If we sell "observability," we are a worse Langfuse/Braintrust/Arize and it's a margin race. Our wedge is the **closed self-improvement loop** (eval campaign → reflective `gepaDriver` prompt optimization → gated promotion → Bradley-Terry tournaments → predictive-validity calibration). Observability is the *byproduct*, not the pitch.
+
+**The pitch:** "Plug us in. Your agent runs a closed self-improvement loop against your own use case, gated so it never ships a regression — and you get the eval + trace observability for free as it does."
+
+### Backend-agnostic by design — the sandbox is swappable, not required
+
+The whole stack already runs **without our sandbox**, at two granularities:
+- **`agent-eval`** (the self-improvement engine) touches the sandbox only as a *type* — the LAND tier needs no sandbox.
+- **`agent-runtime`** is runtime-decoupled too: every sandbox import is `import type`, and the sandbox backend takes an *injected* instance. Its `Backend` seam already ships `createOpenAICompatibleBackend` (pure LLM) and `createIterableBackend` (bring-your-own) next to `createSandboxPromptBackend`. A builder runs the full runtime + self-improvement loop on **their own execution backend**, no Tangle sandbox installed.
+
+So adoption is *graduated*, and the builder picks the depth: (1) **trace-analysis only** over their existing runtime; (2) the **full self-improvement loop** on their own backend; (3) **our sandbox as a swap-in backend** for the batteries. Our sandbox becomes the best *option*, never the cost of entry. The only work to make this explicit: mark the `@tangle-network/sandbox` peer `optional` in agent-runtime + document the `Backend` interface as the swap seam (Phase A).
+
+## The surface — three tiers (land → expand → platform)
+
+| Tier | What they do | What they get | Billing |
+|---|---|---|---|
+| **LAND** (exists today) | `npm i @tangle-network/agent-eval`, wrap their agent behind one `dispatch` seam, bring a judge | Full self-improvement loop + **local** trace/eval artifacts. Any infra, no sandbox. | Free (lib) |
+| **EXPAND** (the build) | Route trace/eval/labeled-scenario data to our orchestrator | Hosted dashboards, cross-run intelligence, the capture flywheel as a service | **Metered** — composes with existing sandbox Stripe + cost-ledger |
+| **PLATFORM** (the carrot) | Move execution into our sandbox (agent-dev-container) | Substrate + orchestrator data/intelligence pre-wired, batteries included | Sandbox usage |
+
+The free lib casts the widest possible net at near-zero cost (it's already published). Value capture is EXPAND: hosting their data/intelligence = a billable surface on the dimensions we already meter (ingested/retained volume, eval-campaign compute, loop runs, seats). "We don't host observability unless they route to us" is the *business model*, not a gap.
+
+## Plan & gates — land-first, validate, then build
+
+The non-negotiable discipline: **do not build the hosted/billing tier before the free LAND is validated on a foreign agent.** Reality on someone else's real agent is cheaper than our imagination.
+
+- **Phase A — unblock (ungated, now):** upgrade agent-dev-container to the latest substrate; export `OutcomeStore`/`DeploymentOutcome` from `/rl`; mark agent-runtime's `@tangle-network/sandbox` peer `optional` + document the `Backend` swap seam (make sandbox-free adoption explicitly supported). Pure wins, correct regardless of the wedge.
+- **Phase B — design-partner LAND validation (forcing function):** wrap the founder's agent behind `dispatch`, author a marketing-quality judge, run one real campaign + `gepaDriver` loop. Instrument integration friction, judge cold-start, and actual quality lift.
+- **GATE — go/no-go + pricing:** decided from Phase B evidence, not theory.
+- **Phase C — LAND ergonomics:** external 15-minute quickstart + a stable `dispatch`/judge/scenario contract + ≥1 reference framework adapter.
+- **Phase D — EXPAND (gated):** hosted OTLP/eval-run HTTP sink (client in agent-eval) + multi-tenant orchestrator ingest + metered billing + minimal dashboard (server in `@tangle-network/monorepo`).
+
+## Success metrics
+
+- **Phase B:** ≥1 measurable quality lift on the partner's own use case; integration ≤ a 1–2 day pairing.
+- **LAND:** time-to-first-self-improvement-loop for a new external agent < 1 day from the quickstart.
+- **EXPAND:** first external tenant routed + first metered dollar.
+
+## Risk register
+
+**Knowns (high confidence)**
+- The substrate is a portable lib; it wraps our 6 agents behind `dispatch`/judge/gate in production.
+- It's runtime-agnostic (FS + in-memory storage, Node + edge) and emits OTLP traces.
+- **agent-runtime is already sandbox-decoupled at runtime** — sandbox is type-only + dependency-injected; the `Backend` seam (`createOpenAICompatibleBackend` / `createIterableBackend` / `createSandboxPromptBackend`) makes the sandbox one swappable option. Only the peer-dep label needs to change.
+- The orchestrator/observability/billing platform lives in `@tangle-network/monorepo`.
+
+**Known-unknowns (Phase B answers these)**
+- Does `dispatch` wrap a *foreign* agent as cleanly as ours, or are there hidden Tangle assumptions?
+- Judge cold-start: can a team with no prior evals author a usable judge for a subjective domain (marketing quality)?
+- Data cold-start: no evals = no scenarios = no labels; how do they bootstrap the flywheel day 1?
+- Orchestrator multi-tenancy: it's only run internally — external auth/isolation/privacy is unbuilt.
+- Pricing line + OSS boundary (the lib is already public).
+
+**Unknown-unknowns (instrument, don't predict)**
+- Integration-surface explosion → keep a tiny stable contract + reference adapters; refuse to special-case.
+- Foreign-domain eval semantics we've never seen → the design partner is the discovery mechanism.
+- Multi-tenant orchestrator failure modes at external/adversarial scale.
+- Supply-chain/trust — we enter their dependency tree; our security posture becomes their procurement question.
+- *Mitigation for all:* land-first-with-a-partner surfaces surprises cheaply, on a real agent, before any hosted spend. Timebox Phase D behind the gate.
+
+## Non-goals (anti-thrash)
+
+- **Not** a standalone observability product. Observability ships only as a byproduct of self-improvement.
+- **Not** building the hosted/billing tier before Phase B validates the lib.
+- **Not** per-framework bespoke integrations — one stable contract + a couple of reference adapters.
+- **Not** re-architecting the substrate for hypothetical external needs — extend at the `dispatch`/judge/store seams only.
+- **Not** changing the sandbox/products roadmap — this *exposes the same engine*, it doesn't fork it.
+
+## Tracking
+
+ops-board epic **#507** + children: agent-dev-container stack upgrade (eng), foreign-agent adoption surface (eng), design-partner LAND validation (gtm), hosted orchestrator routing + billing (eng, blocked on the gate), GTM-fit + pricing decision (gtm).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.43.1",
+  "version": "0.44.0",
   "description": "Substrate for self-improving agents: traces, verifiable rewards, preferences, GEPA / reflective mutation, auto-research, replay, sequential anytime-valid stats, and release gates.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -104,6 +104,11 @@
       "import": "./dist/campaign/index.js",
       "default": "./dist/campaign/index.js"
     },
+    "./contract": {
+      "types": "./dist/contract/index.d.ts",
+      "import": "./dist/contract/index.js",
+      "default": "./dist/contract/index.js"
+    },
     "./openapi.json": {
       "default": "./dist/openapi.json"
     }
@@ -140,7 +145,7 @@
     "zod": "^4.3.6"
   },
   "peerDependencies": {
-    "@tangle-network/agent-runtime": "^0.21.0",
+    "@tangle-network/agent-runtime": ">=0.21.0 <0.26.0",
     "@tangle-network/sandbox": ">=0.2.1 <0.4.0"
   },
   "peerDependenciesMeta": {
@@ -153,7 +158,7 @@
   },
   "devDependencies": {
     "@biomejs/biome": "^2.4.15",
-    "@tangle-network/agent-runtime": "^0.21.0",
+    "@tangle-network/agent-runtime": ">=0.21.0 <0.26.0",
     "@tangle-network/sandbox": "0.3.0",
     "@types/node": "^25.6.0",
     "husky": "^9.1.7",