@tangle-network/agent-eval 0.44.0 → 0.45.0

package/dist/chunk-J3EIOI3O.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/campaign/run-campaign.ts","../src/campaign/storage.ts"],"sourcesContent":["/**\n * @experimental\n *\n * `runCampaign` — Pass A substrate primitive. ONE function that orchestrates\n * scenarios → dispatch → artifacts → judges → aggregates, with full\n * reproducibility (seed + manifest hash), cell-level resumability, bootstrap\n * CIs, and the `LabeledScenarioStore` capture flywheel.\n *\n * Improvement loops (optimizer / gate / autoOnPromote) ride on top of this\n * primitive but live in `presets/run-improvement-loop.ts`. This file keeps\n * the core orchestrator minimal — Phase 1 of the Pass A track.\n */\n\nimport { createHash } from 'node:crypto'\nimport { join } from 'node:path'\nimport { confidenceInterval } from '../statistics'\nimport { type CampaignStorage, fsCampaignStorage } from './storage'\nimport type {\n  CampaignAggregates,\n  CampaignArtifactWriter,\n  CampaignCellResult,\n  CampaignCostMeter,\n  CampaignResult,\n  CampaignTraceWriter,\n  DispatchContext,\n  DispatchFn,\n  JudgeAggregate,\n  JudgeConfig,\n  JudgeScore,\n  LabeledScenarioStore,\n  Scenario,\n  ScenarioAggregate,\n  TraceSpan,\n} from './types'\n\nexport interface RunCampaignOptions<TScenario extends Scenario, TArtifact> {\n  scenarios: TScenario[]\n  dispatch: DispatchFn<TScenario, TArtifact>\n  judges?: JudgeConfig<TArtifact, TScenario>[]\n  /** Required for reproducibility. Default 42. */\n  seed?: number\n  /** Per-scenario replicates for CI bands. Default 1; raise to 5+ for\n   *  bootstrap-tight intervals on critical eval. */\n  reps?: number\n  /** When true (default), completed cells are cached by\n   *  (manifestHash, scenarioId, rep, generation). Re-runs skip cached cells. */\n  resumable?: boolean\n  /** Optional store — when present, every artifact + judge score is captured\n   *  with the configured `captureSource`. Capture is default ON; pass `'off'`\n   *  to disable. */\n  labeledStore?: LabeledScenarioStore | 'off'\n  captureSource?: 'production-trace' | 'eval-run' | 'manual' | 'red-team' | 'synthetic'\n  captureSourceVersionHash?: string\n  /** Wall-clock cost cap across all cells. Cells beyond ceiling are skipped. */\n  costCeiling?: number\n  /** Max concurrent cells. Default 2. */\n  maxConcurrency?: number\n  /** Required: where artifacts + traces land. */\n  runDir: string\n  /** Tracing posture. Default is the substrate's `FileSystemTraceStore` rooted\n   *  at `<runDir>/traces/`. `'off'` disables capture entirely — substrate\n   *  refuses this when the caller wires `autoOnPromote !== 'none'`. */\n  tracing?: 'on' | 'off'\n  /** Test seam — override the wall clock for deterministic tests. */\n  now?: () => Date\n  /** Test seam — override per-cell trace writer factory. */\n  buildTraceWriter?: (cellId: string, dir: string) => CampaignTraceWriter\n  /** Storage backend for run/cell dirs, the resumability cache, artifacts,\n   *  and trace spans. Default: the Node filesystem (`fsCampaignStorage`).\n   *  Pass `inMemoryCampaignStorage()` to run in a filesystem-less runtime\n   *  (Cloudflare Workers, Deno, edge) — the `CampaignResult` is still\n   *  produced; artifacts/traces just aren't persisted to disk. */\n  storage?: CampaignStorage\n  /**\n   * Optional per-cell placement strategy. Returns an opaque string the\n   * substrate forwards as `ctx.placement` to the Dispatch — placement-aware\n   * Dispatches (e.g. `httpDispatch` from `/adapters/http`) use it to route\n   * each cell to the right worker, region, or sandbox. When unset, every\n   * cell receives `ctx.placement = undefined` and behaves identically to\n   * the in-process case.\n   *\n   * @example\n   *   cellPlacement: ({ scenario }) => scenario.tags?.includes('eu') ? 'eu-west' : 'us-east'\n   */\n  cellPlacement?: (input: {\n    scenario: TScenario\n    rep: number\n    generation?: number\n  }) => string | undefined\n}\n\nexport async function runCampaign<TScenario extends Scenario, TArtifact>(\n  opts: RunCampaignOptions<TScenario, TArtifact>,\n): Promise<CampaignResult<TArtifact, TScenario>> {\n  const seed = opts.seed ?? 42\n  const reps = opts.reps ?? 1\n  const resumable = opts.resumable ?? true\n  const maxConcurrency = opts.maxConcurrency ?? 2\n  const now = opts.now ?? (() => new Date())\n  const judges = opts.judges ?? []\n  const storage = opts.storage ?? fsCampaignStorage()\n\n  storage.ensureDir(opts.runDir)\n\n  const manifestHash = computeManifestHash({\n    scenarios: opts.scenarios,\n    judges: judges as unknown as JudgeConfig<unknown>[],\n    dispatchRef: opts.dispatch.name || 'anonymous',\n    seed,\n    reps,\n  })\n\n  const startedAt = now()\n  const cells: CampaignCellResult<TArtifact>[] = []\n  const artifactsByPath: Record<string, string> = {}\n\n  // Build the cell schedule (scenario × rep).\n  const schedule: Array<{ scenario: TScenario; rep: number; cellId: string; cellSeed: number }> = []\n  let cellIndex = 0\n  for (const scenario of opts.scenarios) {\n    for (let rep = 0; rep < reps; rep++) {\n      const cellId = `${scenario.id}:${rep}`\n      const cellSeed = seed + cellIndex\n      schedule.push({ scenario, rep, cellId, cellSeed })\n      cellIndex += 1\n    }\n  }\n\n  // Concurrency-limited execution.\n  let totalCostUsd = 0\n  let costCeilingReached = false\n  const abortController = new AbortController()\n  // Concurrency lanes that drain the cell schedule. Named \"lanes\" — not\n  // \"workers\" — to avoid clashing with the taxonomy's worker (= the agent\n  // harness in a sandbox, invoked behind `dispatch`). See loop-taxonomy.md.\n  const lanes: Promise<void>[] = []\n  let nextIdx = 0\n  const cellsRef = cells\n\n  for (let i = 0; i < maxConcurrency; i++) {\n    lanes.push(\n      (async () => {\n        while (true) {\n          const myIdx = nextIdx++\n          if (myIdx >= schedule.length) return\n          const slot = schedule[myIdx]!\n          if (costCeilingReached) {\n            cellsRef.push(skippedCell(slot, 'cost_ceiling_reached'))\n            continue\n          }\n          const result = await executeCell({\n            slot,\n            opts,\n            manifestHash,\n            resumable,\n            now,\n            storage,\n            buildTraceWriter: opts.buildTraceWriter ?? defaultBuildTraceWriter(storage),\n            signal: abortController.signal,\n          })\n          cellsRef.push(result.cell)\n          totalCostUsd += result.cell.costUsd\n          Object.assign(artifactsByPath, result.artifactsByPath)\n          if (opts.costCeiling !== undefined && totalCostUsd >= opts.costCeiling) {\n            costCeilingReached = true\n          }\n          // Capture into LabeledScenarioStore unless explicitly disabled.\n          if (opts.labeledStore && opts.labeledStore !== 'off' && !result.cell.error) {\n            await captureToStore({\n              store: opts.labeledStore,\n              cell: result.cell,\n              scenario: slot.scenario,\n              opts,\n              now,\n            }).catch((err) => {\n              // Capture failures are non-fatal — log but don't crash the campaign.\n              // (Trace would normally land here.)\n              console.warn(\n                `[runCampaign] capture failed for ${result.cell.cellId}: ${err instanceof Error ? err.message : String(err)}`,\n              )\n            })\n          }\n        }\n      })(),\n    )\n  }\n  await Promise.all(lanes)\n\n  const endedAt = now()\n  cellsRef.sort((a, b) => a.cellId.localeCompare(b.cellId))\n\n  const aggregates = computeAggregates(\n    cellsRef,\n    judges as unknown as JudgeConfig<TArtifact>[],\n    seed,\n  )\n\n  return {\n    manifestHash,\n    seed,\n    startedAt: startedAt.toISOString(),\n    endedAt: endedAt.toISOString(),\n    durationMs: endedAt.getTime() - startedAt.getTime(),\n    cells: cellsRef,\n    aggregates,\n    runDir: opts.runDir,\n    artifactsByPath,\n    scenarios: opts.scenarios.map((s) => ({ id: s.id, kind: s.kind })),\n  }\n}\n\n// ── Internals ─────────────────────────────────────────────────────────\n\ninterface ExecuteCellArgs<TScenario extends Scenario, TArtifact> {\n  slot: { scenario: TScenario; rep: number; cellId: string; cellSeed: number }\n  opts: RunCampaignOptions<TScenario, TArtifact>\n  manifestHash: string\n  resumable: boolean\n  now: () => Date\n  storage: CampaignStorage\n  buildTraceWriter: (cellId: string, dir: string) => CampaignTraceWriter\n  signal: AbortSignal\n}\n\nasync function executeCell<TScenario extends Scenario, TArtifact>(\n  args: ExecuteCellArgs<TScenario, TArtifact>,\n): Promise<{ cell: CampaignCellResult<TArtifact>; artifactsByPath: Record<string, string> }> {\n  const storage = args.storage\n  const cellDir = join(args.opts.runDir, args.slot.cellId.replace(/[^a-zA-Z0-9_-]/g, '_'))\n  storage.ensureDir(cellDir)\n\n  // Resumability: cache key = (manifestHash, scenarioId, rep)\n  const cachePath = join(cellDir, 'cached-result.json')\n  if (args.resumable) {\n    const raw = storage.read(cachePath)\n    if (raw !== undefined) {\n      try {\n        const cached = JSON.parse(raw) as CampaignCellResult<TArtifact>\n        if (cached.cellId === args.slot.cellId) {\n          return { cell: { ...cached, cached: true }, artifactsByPath: {} }\n        }\n      } catch {\n        // Corrupt cache — fall through to re-run.\n      }\n    }\n  }\n\n  const startMs = Date.now()\n  const trace = args.buildTraceWriter(args.slot.cellId, cellDir)\n  const artifactsByPath: Record<string, string> = {}\n  const artifacts: CampaignArtifactWriter = {\n    async write(path, content) {\n      const fullPath = join(cellDir, path)\n      storage.ensureDir(join(fullPath, '..'))\n      storage.write(fullPath, content)\n      artifactsByPath[`${args.slot.cellId}/${path}`] = fullPath\n      return fullPath\n    },\n    async writeJson(path, value) {\n      return artifacts.write(path, JSON.stringify(value, null, 2))\n    },\n  }\n  let costSoFar = 0\n  const cost: CampaignCostMeter = {\n    observe(amount, source) {\n      costSoFar += amount\n      trace.span(`cost.${source}`, { amountUsd: amount }).end()\n    },\n    current() {\n      return costSoFar\n    },\n  }\n\n  const placement = args.opts.cellPlacement?.({\n    scenario: args.slot.scenario,\n    rep: args.slot.rep,\n  })\n\n  const ctx: DispatchContext = {\n    cellId: args.slot.cellId,\n    rep: args.slot.rep,\n    seed: args.slot.cellSeed,\n    signal: args.signal,\n    trace,\n    artifacts,\n    cost,\n    placement,\n  }\n\n  let artifact: TArtifact | undefined\n  let errorMessage: string | undefined\n  try {\n    artifact = await args.opts.dispatch(args.slot.scenario, ctx)\n  } catch (err) {\n    errorMessage = err instanceof Error ? err.message : String(err)\n  }\n\n  // Run judges (only if we have an artifact). A judge that throws invalidates\n  // the cell — recorded as `error`, NOT folded into a fake composite:0 (a fake\n  // zero is indistinguishable from a real zero and poisons every aggregate).\n  const judgeScores: Record<string, JudgeScore> = {}\n  if (artifact !== undefined) {\n    for (const judge of args.opts.judges ?? []) {\n      if (judge.appliesTo && !judge.appliesTo(args.slot.scenario)) continue\n      try {\n        judgeScores[judge.name] = await runJudgeCell(judge, {\n          artifact,\n          scenario: args.slot.scenario,\n          signal: args.signal,\n        })\n      } catch (err) {\n        errorMessage = `judge '${judge.name}' failed: ${err instanceof Error ? err.message : String(err)}`\n        break\n      }\n    }\n  }\n\n  await trace.flush()\n\n  const cell: CampaignCellResult<TArtifact> = {\n    cellId: args.slot.cellId,\n    scenarioId: args.slot.scenario.id,\n    rep: args.slot.rep,\n    artifact: (artifact ?? null) as TArtifact,\n    judgeScores,\n    costUsd: costSoFar,\n    durationMs: Date.now() - startMs,\n    seed: args.slot.cellSeed,\n    cached: false,\n    error: errorMessage,\n  }\n\n  if (!errorMessage && args.resumable) {\n    storage.write(cachePath, JSON.stringify(cell))\n  }\n\n  return { cell, artifactsByPath }\n}\n\nasync function runJudgeCell<TArtifact, TScenario extends Scenario>(\n  judge: JudgeConfig<TArtifact, TScenario>,\n  input: { artifact: TArtifact; scenario: TScenario; signal: AbortSignal },\n): Promise<JudgeScore> {\n  return judge.score(input)\n}\n\nfunction defaultBuildTraceWriter(\n  storage: CampaignStorage,\n): (cellId: string, dir: string) => CampaignTraceWriter {\n  return (cellId, dir) => {\n    const spans: Array<Record<string, unknown>> = []\n    return {\n      span(name, attributes) {\n        const startMs = Date.now()\n        const record: Record<string, unknown> = { name, cellId, startMs, ...(attributes ?? {}) }\n        const finish: TraceSpan = {\n          end(endAttrs) {\n            record.durationMs = Date.now() - startMs\n            if (endAttrs) Object.assign(record, endAttrs)\n            spans.push(record)\n          },\n          setAttribute(key, value) {\n            record[key] = value\n          },\n        }\n        return finish\n      },\n      async flush() {\n        storage.write(join(dir, 'spans.jsonl'), spans.map((s) => JSON.stringify(s)).join('\\n'))\n      },\n    }\n  }\n}\n\nfunction skippedCell<TScenario extends Scenario, TArtifact>(\n  slot: { scenario: TScenario; rep: number; cellId: string; cellSeed: number },\n  reason: string,\n): CampaignCellResult<TArtifact> {\n  return {\n    cellId: slot.cellId,\n    scenarioId: slot.scenario.id,\n    rep: slot.rep,\n    artifact: null as unknown as TArtifact,\n    judgeScores: {},\n    costUsd: 0,\n    durationMs: 0,\n    seed: slot.cellSeed,\n    cached: false,\n    error: `skipped: ${reason}`,\n  }\n}\n\ninterface CaptureArgs<TScenario extends Scenario, TArtifact> {\n  store: LabeledScenarioStore\n  cell: CampaignCellResult<TArtifact>\n  scenario: TScenario\n  opts: RunCampaignOptions<TScenario, TArtifact>\n  now: () => Date\n}\n\nasync function captureToStore<TScenario extends Scenario, TArtifact>(\n  args: CaptureArgs<TScenario, TArtifact>,\n): Promise<void> {\n  await args.store.observe({\n    scenario: args.scenario,\n    artifact: args.cell.artifact,\n    judgeScores: args.cell.judgeScores,\n    source: args.opts.captureSource ?? 'eval-run',\n    sourceVersionHash: args.opts.captureSourceVersionHash ?? 'unknown',\n    capturedAt: args.now().toISOString(),\n    redactionStatus: 'raw',\n  })\n}\n\n// ── Aggregates + manifest hash ────────────────────────────────────────\n\nfunction computeManifestHash(input: {\n  scenarios: Scenario[]\n  judges: JudgeConfig<unknown>[]\n  dispatchRef: string\n  seed: number\n  reps: number\n}): string {\n  const canonical = {\n    scenarios: input.scenarios.map((s) => ({ id: s.id, kind: s.kind })),\n    judges: input.judges.map((j) => ({ name: j.name, dims: j.dimensions.map((d) => d.key) })),\n    dispatch: input.dispatchRef,\n    seed: input.seed,\n    reps: input.reps,\n  }\n  return createHash('sha256').update(JSON.stringify(canonical)).digest('hex')\n}\n\nfunction computeAggregates<TArtifact>(\n  cells: CampaignCellResult<TArtifact>[],\n  judges: JudgeConfig<TArtifact>[],\n  seed: number,\n): CampaignAggregates {\n  const byJudge: Record<string, JudgeAggregate> = {}\n  for (const judge of judges) {\n    const scores: number[] = []\n    for (const cell of cells) {\n      const s = cell.judgeScores[judge.name]\n      if (s !== undefined) scores.push(s.composite)\n    }\n    byJudge[judge.name] = aggregate(scores, seed)\n  }\n  const byScenario: Record<string, ScenarioAggregate> = {}\n  const scenarioGroups = new Map<string, number[]>()\n  for (const cell of cells) {\n    const composites = Object.values(cell.judgeScores).map((s) => s.composite)\n    if (composites.length === 0) continue\n    const mean = composites.reduce((a, b) => a + b, 0) / composites.length\n    const arr = scenarioGroups.get(cell.scenarioId) ?? []\n    arr.push(mean)\n    scenarioGroups.set(cell.scenarioId, arr)\n  }\n  for (const [scenarioId, samples] of scenarioGroups) {\n    const ag = aggregate(samples, seed)\n    byScenario[scenarioId] = { meanComposite: ag.mean, ci95: ag.ci95, n: ag.n }\n  }\n  return {\n    byJudge,\n    byScenario,\n    totalCostUsd: cells.reduce((a, c) => a + c.costUsd, 0),\n    cellsExecuted: cells.filter((c) => !c.error).length,\n    cellsSkipped: cells.filter((c) => c.error?.startsWith('skipped:')).length,\n    cellsCached: cells.filter((c) => c.cached).length,\n    cellsFailed: cells.filter((c) => c.error && !c.error.startsWith('skipped:')).length,\n  }\n}\n\n// Percentile bootstrap CI95 via seeded resampling. Deterministic for a given\n// seed — same campaign re-run produces identical CI bands. Falls back to\n// degenerate intervals at n<=1 (the bootstrap is undefined there).\nfunction aggregate(samples: number[], seed: number): JudgeAggregate {\n  const n = samples.length\n  if (n === 0) return { mean: 0, stdev: 0, ci95: [0, 0], n: 0 }\n  const mean = samples.reduce((a, b) => a + b, 0) / n\n  const variance = samples.reduce((a, b) => a + (b - mean) ** 2, 0) / Math.max(1, n - 1)\n  const stdev = Math.sqrt(variance)\n  const ci = confidenceInterval(samples, 0.95, { seed, resamples: 1000 })\n  return { mean, stdev, ci95: [ci.lower, ci.upper], n }\n}\n","/**\n * @experimental\n *\n * `CampaignStorage` — the filesystem seam `runCampaign` writes through\n * (run/cell dirs, the resumability cache, per-cell artifacts, trace spans).\n *\n * The default (`fsCampaignStorage`) is the Node filesystem — identical\n * behavior to the inline `node:fs` calls it replaces, so existing CLI\n * consumers are unaffected. `inMemoryCampaignStorage` keeps everything in a\n * `Map`, so the substrate runs in environments WITHOUT a filesystem\n * (Cloudflare Workers, Deno Deploy, other edge runtimes) — the campaign\n * still produces its `CampaignResult` (cells + aggregates) in memory;\n * artifacts/traces simply aren't persisted to disk.\n *\n * Paths are opaque keys to the in-memory adapter — it does not parse them,\n * so the same `join(...)`-built paths work unchanged across both adapters.\n */\nexport interface CampaignStorage {\n  /** Ensure a directory exists (recursive). No-op for in-memory. */\n  ensureDir(dir: string): void\n  /** Does this path exist (as a written file or an ensured dir)? */\n  exists(path: string): boolean\n  /** Read a UTF-8 file; `undefined` when missing or unreadable. */\n  read(path: string): string | undefined\n  /** Write a file (string or bytes). Parent dir is assumed ensured. */\n  write(path: string, content: string | Uint8Array): void\n}\n\n/** Node-filesystem storage — the default. Lazily requires `node:fs` so the\n *  module imports cleanly in non-Node runtimes (where the caller passes\n *  `inMemoryCampaignStorage` instead and never constructs this). */\nexport function fsCampaignStorage(): CampaignStorage {\n  const { existsSync, mkdirSync, readFileSync, writeFileSync } =\n    require('node:fs') as typeof import('node:fs')\n  return {\n    ensureDir(dir) {\n      if (!existsSync(dir)) mkdirSync(dir, { recursive: true })\n    },\n    exists(path) {\n      return existsSync(path)\n    },\n    read(path) {\n      try {\n        return readFileSync(path, 'utf8')\n      } catch {\n        return undefined\n      }\n    },\n    write(path, content) {\n      writeFileSync(path, content as Uint8Array)\n    },\n  }\n}\n\n/** In-memory storage for filesystem-less runtimes. Artifacts + trace spans\n *  live in a `Map` for the duration of the run; the `CampaignResult` is\n *  fully populated, but nothing is persisted to disk. */\nexport function inMemoryCampaignStorage(): CampaignStorage {\n  const files = new Map<string, string | Uint8Array>()\n  const dirs = new Set<string>()\n  return {\n    ensureDir(dir) {\n      dirs.add(dir)\n    },\n    exists(path) {\n      return files.has(path) || dirs.has(path)\n    },\n    read(path) {\n      const value = files.get(path)\n      if (value === undefined) return undefined\n      return typeof value === 'string' ? value : new TextDecoder().decode(value)\n    },\n    write(path, content) {\n      files.set(path, content)\n    },\n  }\n}\n"],"mappings":";;;;;;;;AAaA,SAAS,kBAAkB;AAC3B,SAAS,YAAY;;;ACiBd,SAAS,oBAAqC;AACnD,QAAM,EAAE,YAAY,WAAW,cAAc,cAAc,IACzD,UAAQ,IAAS;AACnB,SAAO;AAAA,IACL,UAAU,KAAK;AACb,UAAI,CAAC,WAAW,GAAG,EAAG,WAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AAAA,IAC1D;AAAA,IACA,OAAO,MAAM;AACX,aAAO,WAAW,IAAI;AAAA,IACxB;AAAA,IACA,KAAK,MAAM;AACT,UAAI;AACF,eAAO,aAAa,MAAM,MAAM;AAAA,MAClC,QAAQ;AACN,eAAO;AAAA,MACT;AAAA,IACF;AAAA,IACA,MAAM,MAAM,SAAS;AACnB,oBAAc,MAAM,OAAqB;AAAA,IAC3C;AAAA,EACF;AACF;AAKO,SAAS,0BAA2C;AACzD,QAAM,QAAQ,oBAAI,IAAiC;AACnD,QAAM,OAAO,oBAAI,IAAY;AAC7B,SAAO;AAAA,IACL,UAAU,KAAK;AACb,WAAK,IAAI,GAAG;AAAA,IACd;AAAA,IACA,OAAO,MAAM;AACX,aAAO,MAAM,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI;AAAA,IACzC;AAAA,IACA,KAAK,MAAM;AACT,YAAM,QAAQ,MAAM,IAAI,IAAI;AAC5B,UAAI,UAAU,OAAW,QAAO;AAChC,aAAO,OAAO,UAAU,WAAW,QAAQ,IAAI,YAAY,EAAE,OAAO,KAAK;AAAA,IAC3E;AAAA,IACA,MAAM,MAAM,SAAS;AACnB,YAAM,IAAI,MAAM,OAAO;AAAA,IACzB;AAAA,EACF;AACF;;;ADeA,eAAsB,YACpB,MAC+C;AAC/C,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,iBAAiB,KAAK,kBAAkB;AAC9C,QAAM,MAAM,KAAK,QAAQ,MAAM,oBAAI,KAAK;AACxC,QAAM,SAAS,KAAK,UAAU,CAAC;AAC/B,QAAM,UAAU,KAAK,WAAW,kBAAkB;AAElD,UAAQ,UAAU,KAAK,MAAM;AAE7B,QAAM,eAAe,oBAAoB;AAAA,IACvC,WAAW,KAAK;AAAA,IAChB;AAAA,IACA,aAAa,KAAK,SAAS,QAAQ;AAAA,IACnC;AAAA,IACA;AAAA,EACF,CAAC;AAED,QAAM,YAAY,IAAI;AACtB,QAAM,QAAyC,CAAC;AAChD,QAAM,kBAA0C,CAAC;AAGjD,QAAM,WAA0F,CAAC;AACjG,MAAI,YAAY;AAChB,aAAW,YAAY,KAAK,WAAW;AACrC,aAAS,MAAM,GAAG,MAAM,MAAM,OAAO;AACnC,YAAM,SAAS,GAAG,SAAS,EAAE,IAAI,GAAG;AACpC,YAAM,WAAW,OAAO;AACxB,eAAS,KAAK,EAAE,UAAU,KAAK,QAAQ,SAAS,CAAC;AACjD,mBAAa;AAAA,IACf;AAAA,EACF;AAGA,MAAI,eAAe;AACnB,MAAI,qBAAqB;AACzB,QAAM,kBAAkB,IAAI,gBAAgB;AAI5C,QAAM,QAAyB,CAAC;AAChC,MAAI,UAAU;AACd,QAAM,WAAW;AAEjB,WAAS,IAAI,GAAG,IAAI,gBAAgB,KAAK;AACvC,UAAM;AAAA,OACH,YAAY;AACX,eAAO,MAAM;AACX,gBAAM,QAAQ;AACd,cAAI,SAAS,SAAS,OAAQ;AAC9B,gBAAM,OAAO,SAAS,KAAK;AAC3B,cAAI,oBAAoB;AACtB,qBAAS,KAAK,YAAY,MAAM,sBAAsB,CAAC;AACvD;AAAA,UACF;AACA,gBAAM,SAAS,MAAM,YAAY;AAAA,YAC/B;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,kBAAkB,KAAK,oBAAoB,wBAAwB,OAAO;AAAA,YAC1E,QAAQ,gBAAgB;AAAA,UAC1B,CAAC;AACD,mBAAS,KAAK,OAAO,IAAI;AACzB,0BAAgB,OAAO,KAAK;AAC5B,iBAAO,OAAO,iBAAiB,OAAO,eAAe;AACrD,cAAI,KAAK,gBAAgB,UAAa,gBAAgB,KAAK,aAAa;AACtE,iCAAqB;AAAA,UACvB;AAEA,cAAI,KAAK,gBAAgB,KAAK,iBAAiB,SAAS,CAAC,OAAO,KAAK,OAAO;AAC1E,kBAAM,eAAe;AAAA,cACnB,OAAO,KAAK;AAAA,cACZ,MAAM,OAAO;AAAA,cACb,UAAU,KAAK;AAAA,cACf;AAAA,cACA;AAAA,YACF,CAAC,EAAE,MAAM,CAAC,QAAQ;AAGhB,sBAAQ;AAAA,gBACN,oCAAoC,OAAO,KAAK,MAAM,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,cAC7G;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF,GAAG;AAAA,IACL;AAAA,EACF;AACA,QAAM,QAAQ,IAAI,KAAK;AAEvB,QAAM,UAAU,IAAI;AACpB,WAAS,KAAK,CAAC,GAAG,MAAM,EAAE,OAAO,cAAc,EAAE,MAAM,CAAC;AAExD,QAAM,aAAa;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,WAAW,UAAU,YAAY;AAAA,IACjC,SAAS,QAAQ,YAAY;AAAA,IAC7B,YAAY,QAAQ,QAAQ,IAAI,UAAU,QAAQ;AAAA,IAClD,OAAO;AAAA,IACP;AAAA,IACA,QAAQ,KAAK;AAAA,IACb;AAAA,IACA,WAAW,KAAK,UAAU,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,MAAM,EAAE,KAAK,EAAE;AAAA,EACnE;AACF;AAeA,eAAe,YACb,MAC2F;AAC3F,QAAM,UAAU,KAAK;AACrB,QAAM,UAAU,KAAK,KAAK,KAAK,QAAQ,KAAK,KAAK,OAAO,QAAQ,mBAAmB,GAAG,CAAC;AACvF,UAAQ,UAAU,OAAO;AAGzB,QAAM,YAAY,KAAK,SAAS,oBAAoB;AACpD,MAAI,KAAK,WAAW;AAClB,UAAM,MAAM,QAAQ,KAAK,SAAS;AAClC,QAAI,QAAQ,QAAW;AACrB,UAAI;AACF,cAAM,SAAS,KAAK,MAAM,GAAG;AAC7B,YAAI,OAAO,WAAW,KAAK,KAAK,QAAQ;AACtC,iBAAO,EAAE,MAAM,EAAE,GAAG,QAAQ,QAAQ,KAAK,GAAG,iBAAiB,CAAC,EAAE;AAAA,QAClE;AAAA,MACF,QAAQ;AAAA,MAER;AAAA,IACF;AAAA,EACF;AAEA,QAAM,UAAU,KAAK,IAAI;AACzB,QAAM,QAAQ,KAAK,iBAAiB,KAAK,KAAK,QAAQ,OAAO;AAC7D,QAAM,kBAA0C,CAAC;AACjD,QAAM,YAAoC;AAAA,IACxC,MAAM,MAAM,MAAM,SAAS;AACzB,YAAM,WAAW,KAAK,SAAS,IAAI;AACnC,cAAQ,UAAU,KAAK,UAAU,IAAI,CAAC;AACtC,cAAQ,MAAM,UAAU,OAAO;AAC/B,sBAAgB,GAAG,KAAK,KAAK,MAAM,IAAI,IAAI,EAAE,IAAI;AACjD,aAAO;AAAA,IACT;AAAA,IACA,MAAM,UAAU,MAAM,OAAO;AAC3B,aAAO,UAAU,MAAM,MAAM,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,IAC7D;AAAA,EACF;AACA,MAAI,YAAY;AAChB,QAAM,OAA0B;AAAA,IAC9B,QAAQ,QAAQ,QAAQ;AACtB,mBAAa;AACb,YAAM,KAAK,QAAQ,MAAM,IAAI,EAAE,WAAW,OAAO,CAAC,EAAE,IAAI;AAAA,IAC1D;AAAA,IACA,UAAU;AACR,aAAO;AAAA,IACT;AAAA,EACF;AAEA,QAAM,YAAY,KAAK,KAAK,gBAAgB;AAAA,IAC1C,UAAU,KAAK,KAAK;AAAA,IACpB,KAAK,KAAK,KAAK;AAAA,EACjB,CAAC;AAED,QAAM,MAAuB;AAAA,IAC3B,QAAQ,KAAK,KAAK;AAAA,IAClB,KAAK,KAAK,KAAK;AAAA,IACf,MAAM,KAAK,KAAK;AAAA,IAChB,QAAQ,KAAK;AAAA,IACb;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,eAAW,MAAM,KAAK,KAAK,SAAS,KAAK,KAAK,UAAU,GAAG;AAAA,EAC7D,SAAS,KAAK;AACZ,mBAAe,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,EAChE;AAKA,QAAM,cAA0C,CAAC;AACjD,MAAI,aAAa,QAAW;AAC1B,eAAW,SAAS,KAAK,KAAK,UAAU,CAAC,GAAG;AAC1C,UAAI,MAAM,aAAa,CAAC,MAAM,UAAU,KAAK,KAAK,QAAQ,EAAG;AAC7D,UAAI;AACF,oBAAY,MAAM,IAAI,IAAI,MAAM,aAAa,OAAO;AAAA,UAClD;AAAA,UACA,UAAU,KAAK,KAAK;AAAA,UACpB,QAAQ,KAAK;AAAA,QACf,CAAC;AAAA,MACH,SAAS,KAAK;AACZ,uBAAe,UAAU,MAAM,IAAI,aAAa,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAChG;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,MAAM,MAAM;AAElB,QAAM,OAAsC;AAAA,IAC1C,QAAQ,KAAK,KAAK;AAAA,IAClB,YAAY,KAAK,KAAK,SAAS;AAAA,IAC/B,KAAK,KAAK,KAAK;AAAA,IACf,UAAW,YAAY;AAAA,IACvB;AAAA,IACA,SAAS;AAAA,IACT,YAAY,KAAK,IAAI,IAAI;AAAA,IACzB,MAAM,KAAK,KAAK;AAAA,IAChB,QAAQ;AAAA,IACR,OAAO;AAAA,EACT;AAEA,MAAI,CAAC,gBAAgB,KAAK,WAAW;AACnC,YAAQ,MAAM,WAAW,KAAK,UAAU,IAAI,CAAC;AAAA,EAC/C;AAEA,SAAO,EAAE,MAAM,gBAAgB;AACjC;AAEA,eAAe,aACb,OACA,OACqB;AACrB,SAAO,MAAM,MAAM,KAAK;AAC1B;AAEA,SAAS,wBACP,SACsD;AACtD,SAAO,CAAC,QAAQ,QAAQ;AACtB,UAAM,QAAwC,CAAC;AAC/C,WAAO;AAAA,MACL,KAAK,MAAM,YAAY;AACrB,cAAM,UAAU,KAAK,IAAI;AACzB,cAAM,SAAkC,EAAE,MAAM,QAAQ,SAAS,GAAI,cAAc,CAAC,EAAG;AACvF,cAAM,SAAoB;AAAA,UACxB,IAAI,UAAU;AACZ,mBAAO,aAAa,KAAK,IAAI,IAAI;AACjC,gBAAI,SAAU,QAAO,OAAO,QAAQ,QAAQ;AAC5C,kBAAM,KAAK,MAAM;AAAA,UACnB;AAAA,UACA,aAAa,KAAK,OAAO;AACvB,mBAAO,GAAG,IAAI;AAAA,UAChB;AAAA,QACF;AACA,eAAO;AAAA,MACT;AAAA,MACA,MAAM,QAAQ;AACZ,gBAAQ,MAAM,KAAK,KAAK,aAAa,GAAG,MAAM,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,MACxF;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,YACP,MACA,QAC+B;AAC/B,SAAO;AAAA,IACL,QAAQ,KAAK;AAAA,IACb,YAAY,KAAK,SAAS;AAAA,IAC1B,KAAK,KAAK;AAAA,IACV,UAAU;AAAA,IACV,aAAa,CAAC;AAAA,IACd,SAAS;AAAA,IACT,YAAY;AAAA,IACZ,MAAM,KAAK;AAAA,IACX,QAAQ;AAAA,IACR,OAAO,YAAY,MAAM;AAAA,EAC3B;AACF;AAUA,eAAe,eACb,MACe;AACf,QAAM,KAAK,MAAM,QAAQ;AAAA,IACvB,UAAU,KAAK;AAAA,IACf,UAAU,KAAK,KAAK;AAAA,IACpB,aAAa,KAAK,KAAK;AAAA,IACvB,QAAQ,KAAK,KAAK,iBAAiB;AAAA,IACnC,mBAAmB,KAAK,KAAK,4BAA4B;AAAA,IACzD,YAAY,KAAK,IAAI,EAAE,YAAY;AAAA,IACnC,iBAAiB;AAAA,EACnB,CAAC;AACH;AAIA,SAAS,oBAAoB,OAMlB;AACT,QAAM,YAAY;AAAA,IAChB,WAAW,MAAM,UAAU,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,MAAM,EAAE,KAAK,EAAE;AAAA,IAClE,QAAQ,MAAM,OAAO,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,MAAM,EAAE,WAAW,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;AAAA,IACxF,UAAU,MAAM;AAAA,IAChB,MAAM,MAAM;AAAA,IACZ,MAAM,MAAM;AAAA,EACd;AACA,SAAO,WAAW,QAAQ,EAAE,OAAO,KAAK,UAAU,SAAS,CAAC,EAAE,OAAO,KAAK;AAC5E;AAEA,SAAS,kBACP,OACA,QACA,MACoB;AACpB,QAAM,UAA0C,CAAC;AACjD,aAAW,SAAS,QAAQ;AAC1B,UAAM,SAAmB,CAAC;AAC1B,eAAW,QAAQ,OAAO;AACxB,YAAM,IAAI,KAAK,YAAY,MAAM,IAAI;AACrC,UAAI,MAAM,OAAW,QAAO,KAAK,EAAE,SAAS;AAAA,IAC9C;AACA,YAAQ,MAAM,IAAI,IAAI,UAAU,QAAQ,IAAI;AAAA,EAC9C;AACA,QAAM,aAAgD,CAAC;AACvD,QAAM,iBAAiB,oBAAI,IAAsB;AACjD,aAAW,QAAQ,OAAO;AACxB,UAAM,aAAa,OAAO,OAAO,KAAK,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS;AACzE,QAAI,WAAW,WAAW,EAAG;AAC7B,UAAM,OAAO,WAAW,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,WAAW;AAChE,UAAM,MAAM,eAAe,IAAI,KAAK,UAAU,KAAK,CAAC;AACpD,QAAI,KAAK,IAAI;AACb,mBAAe,IAAI,KAAK,YAAY,GAAG;AAAA,EACzC;AACA,aAAW,CAAC,YAAY,OAAO,KAAK,gBAAgB;AAClD,UAAM,KAAK,UAAU,SAAS,IAAI;AAClC,eAAW,UAAU,IAAI,EAAE,eAAe,GAAG,MAAM,MAAM,GAAG,MAAM,GAAG,GAAG,EAAE;AAAA,EAC5E;AACA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,SAAS,CAAC;AAAA,IACrD,eAAe,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE;AAAA,IAC7C,cAAc,MAAM,OAAO,CAAC,MAAM,EAAE,OAAO,WAAW,UAAU,CAAC,EAAE;AAAA,IACnE,aAAa,MAAM,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE;AAAA,IAC3C,aAAa,MAAM,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,WAAW,UAAU,CAAC,EAAE;AAAA,EAC/E;AACF;AAKA,SAAS,UAAU,SAAmB,MAA8B;AAClE,QAAM,IAAI,QAAQ;AAClB,MAAI,MAAM,EAAG,QAAO,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,EAAE;AAC5D,QAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;AAClD,QAAM,WAAW,QAAQ,OAAO,CAAC,GAAG,MAAM,KAAK,IAAI,SAAS,GAAG,CAAC,IAAI,KAAK,IAAI,GAAG,IAAI,CAAC;AACrF,QAAM,QAAQ,KAAK,KAAK,QAAQ;AAChC,QAAM,KAAK,mBAAmB,SAAS,MAAM,EAAE,MAAM,WAAW,IAAK,CAAC;AACtE,SAAO,EAAE,MAAM,OAAO,MAAM,CAAC,GAAG,OAAO,GAAG,KAAK,GAAG,EAAE;AACtD;","names":[]}

package/dist/contract/index.d.ts

@@ -1,5 +1,5 @@
-export { C as CampaignAggregates, a as CampaignArtifactWriter, b as CampaignCellResult, c as CampaignCostMeter, d as CampaignResult, e as CampaignTraceWriter, f as CodeSurface, D as Dispatch, g as DispatchContext, G as Gate, h as GateContext, i as GateDecision, j as GateResult, k as GenerationCandidate, l as GenerationRecord, I as ImprovementDriver, J as JudgeConfig, m as JudgeDimension, n as JudgeScore, M as MutableSurface, o as Mutator, O as OptimizerConfig, S as Scenario, p as SessionScript } from '../types-DToGONFA.js';
-export { C as CampaignStorage, D as DefaultProductionGateOptions, E as EvolutionaryDriverOptions, G as GepaDriverOptions, H as HeldOutGateOptions, R as RunCampaignOptions, a as RunEvalOptions, b as RunImprovementLoopOptions, c as RunImprovementLoopResult, d as composeGate, e as defaultProductionGate, f as evolutionaryDriver, g as fsCampaignStorage, h as gepaDriver, i as heldOutGate, j as inMemoryCampaignStorage, r as runCampaign, k as runEval, l as runImprovementLoop } from '../run-improvement-loop-CbilHQAb.js';
+export { C as CampaignAggregates, a as CampaignArtifactWriter, b as CampaignCellResult, c as CampaignCostMeter, d as CampaignResult, e as CampaignTraceWriter, f as CodeSurface, D as Dispatch, g as DispatchContext, G as Gate, h as GateContext, i as GateDecision, j as GateResult, k as GenerationCandidate, l as GenerationRecord, I as ImprovementDriver, J as JudgeConfig, m as JudgeDimension, n as JudgeScore, M as MutableSurface, o as Mutator, O as OptimizerConfig, S as Scenario, p as SessionScript } from '../types-BURGZ8Ug.js';
+export { C as CampaignStorage, D as DefaultProductionGateOptions, E as EvolutionaryDriverOptions, G as GepaDriverOptions, H as HeldOutGateOptions, R as RunCampaignOptions, a as RunEvalOptions, b as RunImprovementLoopOptions, c as RunImprovementLoopResult, d as composeGate, e as defaultProductionGate, f as evolutionaryDriver, g as fsCampaignStorage, h as gepaDriver, i as heldOutGate, j as inMemoryCampaignStorage, r as runCampaign, k as runEval, l as runImprovementLoop } from '../run-improvement-loop-pJ4yrx4X.js';
 export { D as DeploymentOutcome, F as FileSystemOutcomeStore, a as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore, O as OutcomeStore } from '../outcome-store-BxJ3DQKJ.js';
 import '../llm-client-BXVRUZyX.js';
 import '../errors-mje_cKOs.js';

package/dist/contract/index.js

@@ -6,12 +6,12 @@ import {
   heldOutGate,
   runEval,
   runImprovementLoop
-} from "../chunk-H5BGRSN4.js";
+} from "../chunk-HRKOCLQA.js";
 import {
   fsCampaignStorage,
   inMemoryCampaignStorage,
   runCampaign
-} from "../chunk-RXK7FXLV.js";
+} from "../chunk-J3EIOI3O.js";
 import "../chunk-N4SBKEPJ.js";
 import "../chunk-YV7J7X5N.js";
 import {

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.43.2",
+    "version": "0.44.1",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/rl.d.ts

@@ -1,5 +1,5 @@
 import { R as RunRecord, a as RunSplitTag } from './run-record-BGY6bHRh.js';
-import { d as CampaignResult } from './types-DToGONFA.js';
+import { d as CampaignResult } from './types-BURGZ8Ug.js';
 import { V as VerificationReport, R as Researcher, F as FailureMode, S as SteeringChange, E as ExperimentPlan, a as ExperimentResult, b as EvalCampaignResult, c as EvalCampaignOptions } from './researcher-CoJMs2Iz.js';
 export { r as runEvalCampaign } from './researcher-CoJMs2Iz.js';
 import { S as Span, T as TraceStore } from './store-Db2Bv8Cf.js';

package/dist/{run-campaign-GNDO66B4.js → run-campaign-6UEVBPP3.js}

@@ -1,10 +1,10 @@
 import {
   runCampaign
-} from "./chunk-RXK7FXLV.js";
+} from "./chunk-J3EIOI3O.js";
 import "./chunk-WP7SY7AI.js";
 import "./chunk-QYJT52YW.js";
 import "./chunk-NSBPE2FW.js";
 export {
   runCampaign
 };
-//# sourceMappingURL=run-campaign-GNDO66B4.js.map
+//# sourceMappingURL=run-campaign-6UEVBPP3.js.map

package/dist/{run-improvement-loop-CbilHQAb.d.ts → run-improvement-loop-pJ4yrx4X.d.ts}

@@ -1,4 +1,4 @@
-import { S as Scenario, d as CampaignResult, j as GateResult, o as Mutator, I as ImprovementDriver, G as Gate, D as DispatchFn, J as JudgeConfig, L as LabeledScenarioStore, e as CampaignTraceWriter, M as MutableSurface, l as GenerationRecord } from './types-DToGONFA.js';
+import { S as Scenario, d as CampaignResult, j as GateResult, o as Mutator, I as ImprovementDriver, G as Gate, D as DispatchFn, J as JudgeConfig, L as LabeledScenarioStore, e as CampaignTraceWriter, M as MutableSurface, l as GenerationRecord } from './types-BURGZ8Ug.js';
 import { L as LlmClientOptions } from './llm-client-BXVRUZyX.js';
 import { RunRecord } from '@tangle-network/agent-runtime';
 import { R as RedTeamCase } from './red-team-30II1T4o.js';
@@ -267,6 +267,22 @@ interface RunCampaignOptions<TScenario extends Scenario, TArtifact> {
      *  (Cloudflare Workers, Deno, edge) — the `CampaignResult` is still
      *  produced; artifacts/traces just aren't persisted to disk. */
     storage?: CampaignStorage;
+    /**
+     * Optional per-cell placement strategy. Returns an opaque string the
+     * substrate forwards as `ctx.placement` to the Dispatch — placement-aware
+     * Dispatches (e.g. `httpDispatch` from `/adapters/http`) use it to route
+     * each cell to the right worker, region, or sandbox. When unset, every
+     * cell receives `ctx.placement = undefined` and behaves identically to
+     * the in-process case.
+     *
+     * @example
+     *   cellPlacement: ({ scenario }) => scenario.tags?.includes('eu') ? 'eu-west' : 'us-east'
+     */
+    cellPlacement?: (input: {
+        scenario: TScenario;
+        rep: number;
+        generation?: number;
+    }) => string | undefined;
 }
 declare function runCampaign<TScenario extends Scenario, TArtifact>(opts: RunCampaignOptions<TScenario, TArtifact>): Promise<CampaignResult<TArtifact, TScenario>>;
 

package/dist/{types-DToGONFA.d.ts → types-BURGZ8Ug.d.ts}

@@ -40,6 +40,14 @@ interface DispatchContext {
     cycleId?: string;
     /** Populated when the substrate resumed from a prior cache hit. */
     resumedFrom?: string;
+    /**
+     * Opaque placement key supplied by `RunCampaignOptions.cellPlacement`.
+     * The substrate forwards it through unchanged; placement-aware Dispatch
+     * implementations (e.g. `httpDispatch` from `/adapters/http`) read it to
+     * route the cell to the right worker / region / sandbox. `undefined`
+     * when no placement strategy is configured.
+     */
+    placement?: string;
 }
 /** @experimental One function: scenario + ctx → artifact. Dispatcher chooses
  *  whether to call `runMultishot`, `runLoop`, raw `streamPrompt`, anything. */

package/docs/adapters-observability.md

@@ -0,0 +1,121 @@
+# Composing agent-eval with your observability stack
+
+`@tangle-network/agent-eval` ships its own OpenTelemetry pipeline
+(`@tangle-network/agent-eval/telemetry`) that emits spans for every
+cell, judge invocation, mutator proposal, and gate decision. **It's
+just OTel** — same protocol as Langfuse SDK, OpenLLMetry, Arize
+Phoenix, TraceAI, and the OpenTelemetry GenAI semantic conventions.
+
+That means: if you already instrument your agent with any OTel-native
+observability tool, the two compose **for free at the protocol layer**.
+This doc shows the composition pattern; no agent-eval-specific adapter
+code required.
+
+## TL;DR — one OTel context, two emitters
+
+1. Set up a shared OTel tracer provider in your process (or service mesh).
+2. Configure your observability tool (TraceAI / Langfuse / OpenLLMetry /
+   Phoenix) to register its instrumentations against that provider.
+3. Configure agent-eval's `/telemetry` exporter against the same provider.
+4. Run a campaign. Both sets of spans land at your OTel collector.
+5. Filter / route / fan-out at the collector layer — Jaeger, Tempo,
+   Phoenix, Langfuse cloud, your private collector, whatever.
+
+The Tangle substrate doesn't compete with the observability tool;
+they're orthogonal. The tool tells you *what your agent did*; the
+substrate tells you *what the campaign / judge / mutator decided about
+it*. Unified at the trace level, you see both as one timeline per cell.
+
+## Per-tool notes
+
+### TraceAI (Future-AGI)
+
+- TS SDK auto-instruments OpenAI/Anthropic SDKs + LangChain.
+- Compatible with the OpenTelemetry GenAI semantic conventions.
+- Compose: register TraceAI's instrumentations on the global tracer
+  provider, then either point both at your OTLP collector or at
+  TraceAI's hosted backend if you want their UI.
+
+### Langfuse SDK
+
+- Larger installed base; has its own hosted product + OSS self-host.
+- Their OpenTelemetry-compatible mode ships LLM call spans with
+  Langfuse-specific attributes preserved.
+- Compose: register Langfuse as an OTel processor; agent-eval's
+  campaign/judge/mutator spans appear alongside the LLM calls in their
+  UI.
+
+### OpenLLMetry (Traceloop)
+
+- OSS auto-instrumentation library; OTel-native by design.
+- Wide framework coverage (LangChain, LlamaIndex, Haystack, OpenAI,
+  Anthropic).
+- Compose: set up Traceloop's exporter; agent-eval's exporter shares
+  the same trace context per cell.
+
+### Arize Phoenix
+
+- OSS observability backend; strong in the eval-tooling community.
+- OTel-native ingest; renders trace + span attributes per the GenAI
+  semantic conventions.
+- Compose: point both exporters at your local Phoenix instance. Phoenix
+  becomes the unified UI for both LLM-call traces and campaign spans.
+
+## Wiring pattern (reference)
+
+```ts
+import { trace } from '@opentelemetry/api'
+import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'
+import { SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base'
+
+// 1. One shared tracer provider for the process.
+const provider = new NodeTracerProvider()
+provider.addSpanProcessor(new SimpleSpanProcessor(
+  new OTLPTraceExporter({ url: 'http://localhost:4318/v1/traces' }),
+))
+provider.register()
+
+// 2. Your observability tool registers against the global provider.
+//    Example for TraceAI / OpenLLMetry / Langfuse — call their init.
+//    (See each tool's docs.)
+
+// 3. agent-eval is already OTel-native; it picks up the same global
+//    provider. Just ensure `@tangle-network/agent-eval/telemetry` is
+//    initialized for the campaign:
+import { setOtelExporter } from '@tangle-network/agent-eval/telemetry'
+setOtelExporter({ kind: 'otel-global' })  // use the global provider
+
+// 4. Run your campaign — both sets of spans land at the collector.
+import { runEval } from '@tangle-network/agent-eval/contract'
+await runEval({ /* ... */ })
+```
+
+That's it. No new adapter shipping required — the libs are already
+designed to live in the same OTel ecosystem.
+
+## When you'd want a deeper, code-level adapter
+
+The two cases where a thin adapter would add value beyond the
+OTel-protocol composition:
+
+1. **Cost-aware judging.** Your observability tool's auto-instrumented
+   spans carry token counts + cost. A custom `JudgeConfig` can read
+   them via the OTel context and refuse to score artifacts that
+   exceeded a per-call budget. Easy to write yourself; we'll ship a
+   reference helper (`costAwareJudgeFromOtel`) when a partner pulls on
+   this.
+2. **Tool-aware judging.** Your instrumentation captures the tool-call
+   sequence (`langchain.tool.invoked`, `openai.function.called`, etc.).
+   A judge that scores "did the agent use the right tool" reads those
+   spans directly. Also straightforward; helper ships when needed.
+
+Both of these are L1-tier ergonomic helpers; the underlying composition
+works today without them.
+
+## What this does NOT install
+
+No new dependencies. No new peer deps. No `@traceai/*`, no
+`@langfuse/*`, no `@opentelemetry/*` in our manifest. You bring the
+observability stack you want; agent-eval just emits OTel and respects
+whatever provider is registered.

package/docs/distributed-driver.md

@@ -0,0 +1,173 @@
+# Distributed driver — driver-on-A, workers-on-B (and C, D, E…)
+
+The driver (running `runCampaign` / `runImprovementLoop` / `gepaDriver`)
+and the worker (running your actual agent) **do not have to live in the
+same process, machine, region, or cloud.** `Dispatch` is just a
+function: scenario in, artifact out. Whatever returns the artifact is
+the worker — local, remote, sandboxed, or fanned out across a fleet.
+
+## Why you'd want this
+
+| Pattern | Reason |
+|---|---|
+| **Driver on your VPC, workers on our sandbox fleet** | Driver holds secrets, training data, prompt corpus; workers stay stateless and scale horizontally |
+| **Multi-region campaigns** | Each cell runs in the region closest to its target API (latency, compliance, data residency) |
+| **Driver-as-a-service** | Long-running optimization process; reuses across many short-lived worker invocations |
+| **Heterogeneous workers** | One cell on a CPU container, another on a GPU box, another against a third-party API — same Dispatch shape, different placement |
+| **Budget-isolated workers** | Worker boxes get scoped, time-bounded credentials; driver never holds production keys |
+
+## Two new pieces in 0.45.0
+
+| Where | What |
+|---|---|
+| **`DispatchContext.placement?: string`** | Opaque placement key the substrate forwards to the Dispatch. |
+| **`RunCampaignOptions.cellPlacement?(input) → string \| undefined`** | Strategy function the substrate calls per cell to compute the placement key. |
+| **`@tangle-network/agent-eval/adapters/http`** | `httpDispatch` (client) + `runDispatchServer` (server) — wire shape for HTTP-based remote workers. |
+
+Both ends of the wire are in the same package; no peer dep, no separate
+install. The substrate doesn't strategy-pick; you provide the
+`cellPlacement` function, the substrate forwards its result, the
+Dispatch reads it. Clean seam, no policy baked in.
+
+## The three reference topologies
+
+### 1. In-process (the default — what you already have)
+
+```ts
+await runCampaign({
+  scenarios,
+  dispatch,            // runs in-process
+  judges: [judge],
+  storage,
+  runDir,
+})
+```
+
+`ctx.placement` is `undefined`; nothing changes for existing consumers.
+This shipped in 0.40.
+
+### 2. Single remote worker
+
+Driver-on-A talks to one worker-on-B over HTTP.
+
+**Driver side (machine A):**
+
+```ts
+import { httpDispatch } from '@tangle-network/agent-eval/adapters/http'
+
+const dispatch = httpDispatch<MyScenario, MyArtifact>({
+  url: 'https://worker.your-infra.com/dispatch',
+  auth: process.env.WORKER_TOKEN,
+  timeoutMs: 5 * 60 * 1000,
+  retries: 2,
+})
+
+await runImprovementLoop({ scenarios, baselineSurface, dispatchWithSurface: (surface, s, ctx) =>
+  dispatch(s, { ...ctx, /* pass the surface through your own protocol */ }),
+  /* ... */ })
+```
+
+**Worker side (machine B):**
+
+```ts
+import { runDispatchServer } from '@tangle-network/agent-eval/adapters/http'
+
+const handle = await runDispatchServer<MyScenario, MyArtifact>({
+  dispatch: async (scenario, ctx) => {
+    // your agent — call OpenAI, LangChain, your sandbox, anything.
+    const artifact = await runMyAgent(scenario, ctx.signal)
+    return artifact
+  },
+  port: 8080,
+  auth: process.env.WORKER_TOKEN,  // required; `false` only for closed networks
+})
+console.log(`worker listening on ${handle.port}`)
+```
+
+Cancellation, retries on 5xx / 408 / 429, bounded timeouts, optional
+custom auth headers, optional `fetchImpl` override — all there.
+
+### 3. Multi-region fan-out
+
+Driver picks a region per cell; the same `httpDispatch` routes to
+different worker URLs based on placement.
+
+```ts
+import { httpDispatch } from '@tangle-network/agent-eval/adapters/http'
+
+const REGION_URLS: Record<string, string> = {
+  'us-east': 'https://worker-use1.your-infra.com/dispatch',
+  'eu-west': 'https://worker-euw1.your-infra.com/dispatch',
+  'ap-south': 'https://worker-aps1.your-infra.com/dispatch',
+}
+
+const dispatch = httpDispatch<MyScenario, MyArtifact>({
+  resolveUrl: ({ placement }) => REGION_URLS[placement ?? 'us-east'],
+  auth: process.env.WORKER_TOKEN,
+})
+
+await runCampaign({
+  scenarios,
+  dispatch,
+  judges: [judge],
+  storage,
+  runDir,
+  cellPlacement: ({ scenario }) => {
+    if (scenario.tags?.includes('eu')) return 'eu-west'
+    if (scenario.tags?.includes('ap')) return 'ap-south'
+    return 'us-east'
+  },
+  maxConcurrency: 8,  // 8 cells fan across regions in parallel
+})
+```
+
+`cellPlacement` is a pure function the substrate calls per cell — no
+state. Use whatever signal you want (tags, hash of scenario id,
+round-robin, region-affinity from a previous run, scheduling table).
+
+## What's preserved across the wire
+
+| Concern | How |
+|---|---|
+| **Cancellation** | Driver's `AbortSignal` forwards into the HTTP request; server translates `AbortError` → `499` so client doesn't retry. |
+| **Timeouts** | Per-call `timeoutMs` on the client; server can layer its own. |
+| **Retries** | Idempotent retries on 5xx / 408 / 429 with exponential backoff + jitter. Driver-aborts never retry. |
+| **Auth** | Bearer token on `Authorization`; pluggable via `auth: string \| () => string \| Promise<string>` for rotation/refresh. |
+| **Payload size** | Server enforces `maxBodyBytes` (default 10 MB). |
+| **Traces** | Both ends emit OTel — if both point at the same OTLP collector, you get a unified trace per cell. See `docs/adapters-observability.md`. |
+| **Cost** | Worker's `ctx.cost.observe(usd, source)` is local to the worker process. Roll up server-side and attach to your worker-side telemetry; we don't (yet) forward cost back to the driver. Tracked as follow-up. |
+
+## Running the reference example
+
+See `examples/distributed-driver/`:
+
+```sh
+# Terminal 1 — worker
+pnpm tsx examples/distributed-driver/worker.ts
+
+# Terminal 2 — driver
+WORKER_URL=http://localhost:8080/dispatch \
+WORKER_TOKEN=dev-token \
+pnpm tsx examples/distributed-driver/driver.ts
+```
+
+Two processes, one local TCP loopback, full self-improvement loop end
+to end. Scaling out is dropping `WORKER_URL` to a non-loopback hostname
+and using `cellPlacement` to fan across many of them.
+
+## Known gaps + follow-ups
+
+- **Cost roll-up across the wire** — worker-side `ctx.cost` observations
+  stay on the worker. We need to forward them in the response body so
+  `defaultProductionGate`'s `budgetUsd` ceiling reflects total spend, not
+  driver-side spend. Tracked as a 0.45.x follow-up.
+- **Per-cell artifact streaming** — when the worker writes intermediate
+  artifacts via `ctx.artifacts.write`, those land on the worker's
+  storage. For multi-worker campaigns you'll want a shared object store
+  (S3/GCS) reachable from both sides; today consumers wire that as a
+  `CampaignStorage` impl. A reference S3-backed storage is on the
+  roadmap.
+- **gRPC / NATS / Temporal transports** — the wire is HTTP today by
+  default because everything speaks HTTP. Other transports can ship as
+  additional adapters; the `Dispatch` interface itself is
+  transport-agnostic.

package/docs/quickstart-external.md

@@ -0,0 +1,190 @@
+# Quickstart — self-improvement loop for any agent (15 minutes)
+
+The standalone walkthrough mirroring
+`examples/foreign-agent-quickstart/`. Read this first; copy the runnable
+example second.
+
+## What you get
+
+After 15 minutes you have a closed self-improvement loop running
+against your agent — measured, gated, and reproducible — with no
+Tangle sandbox, no Tangle account, and no hosted infrastructure.
+
+## Install
+
+```sh
+npm i @tangle-network/agent-eval@^0.44.0
+```
+
+The package's `@tangle-network/sandbox` peer is `optional` (as of
+0.44.0). Foreign consumers can install agent-eval and run the full LAND
+tier without our sandbox or its dependencies.
+
+## Five types, four functions
+
+```ts
+import {
+  // Types
+  type Scenario,        // what you evaluate against (id + kind + your fields)
+  type Dispatch,        // your agent, wrapped as one function
+  type JudgeConfig,     // pluggable dimensional scorer
+  type Mutator,         // proposes a next surface
+  type Gate,            // promotion guard
+
+  // Functions
+  runEval,
+  runCampaign,
+  runImprovementLoop,
+  defaultProductionGate,
+
+  // Storage
+  fsCampaignStorage,
+  inMemoryCampaignStorage,
+} from '@tangle-network/agent-eval/contract'
+```
+
+Every export above is committed under semver. New minors only ADD;
+nothing here changes shape in a 0.x minor.
+
+## Three steps to wire your agent
+
+### 1. Scenarios
+
+```ts
+interface MarketingScenario extends Scenario {
+  blurb: string
+  surface: 'landing-hero' | 'tweet' | 'email-subject'
+  audience: string
+}
+
+const scenarios: MarketingScenario[] = [
+  { id: 's1', kind: 'marketing-rewrite', blurb: '...', surface: 'tweet', audience: '...' },
+  // ...
+]
+```
+
+### 2. Wrap your agent as `Dispatch`
+
+```ts
+const dispatch: Dispatch<MarketingScenario, MarketingArtifact> = async (scenario, ctx) => {
+  const rewrite = await callYourAgent(scenario, { signal: ctx.signal })
+  return { rewrite, modelUsed: '...' }
+}
+```
+
+`ctx` carries `signal` (cancellation), `trace` (write spans), `artifacts`
+(write blobs), `cost` (token + $ meter). Use them or ignore them.
+
+### 3. Bring a judge
+
+```ts
+const judge: JudgeConfig<MarketingArtifact, MarketingScenario> = {
+  name: 'marketing-quality',
+  dimensions: [
+    { key: 'hook_strength', description: '...' },
+    { key: 'voice_match', description: '...' },
+    { key: 'cta_clarity', description: '...' },
+    { key: 'factual_grounding', description: '...' },
+  ],
+  async score({ artifact, scenario, signal }) {
+    // LLM call, heuristic, ensemble — anything. Return JudgeScore.
+    return { dimensions: { ... }, composite: 0.72, notes: '...' }
+  },
+}
+```
+
+Throw on failure; the substrate records it as a failed cell. No silent
+zeros.
+
+## Baseline
+
+```ts
+const baseline = await runEval({
+  scenarios,
+  dispatch,
+  judges: [judge],
+  storage: inMemoryCampaignStorage(),
+  runDir: 'mem://my-baseline',
+})
+
+const score = Object.values(baseline.aggregates.byScenario)
+  .reduce((sum, s) => sum + s.meanComposite, 0) / scenarios.length
+
+console.log(`Baseline composite: ${score.toFixed(3)}`)
+```
+
+## Self-improvement loop
+
+```ts
+import { gepaDriver, defaultProductionGate } from '@tangle-network/agent-eval/contract'
+
+const result = await runImprovementLoop({
+  scenarios: trainScenarios,
+  baselineSurface,
+  dispatchWithSurface: (surface, scenario, ctx) =>
+    runYourAgent({ systemPrompt: surface as string }, scenario, ctx),
+  driver: gepaDriver({
+    llm: { apiKey: process.env.OPENAI_API_KEY, baseUrl: '...' },
+    model: 'gpt-4o-mini',
+    target: 'marketing copywriting system prompt',
+    mutationPrimitives: [
+      'Tighten the hook: lead with the concrete user outcome.',
+      'Replace generic adjectives with specific verbs.',
+      // ...
+    ],
+  }),
+  judges: [judge],
+  populationSize: 2,
+  maxGenerations: 3,
+  holdoutScenarios,
+  gate: defaultProductionGate({
+    holdoutScenarios,
+    deltaThreshold: 0.05,
+  }),
+  autoOnPromote: 'none',
+  storage: inMemoryCampaignStorage(),
+  runDir: 'mem://my-improve',
+})
+
+if (result.gateResult.decision === 'ship') {
+  // Deploy result.winnerSurface — we don't push it for you.
+}
+```
+
+The gate decision is `'ship'` | `'hold'` | `'need_more_work'` |
+`'model_ceiling'` | `'arch_ceiling'`. You define what each means in
+your deploy pipeline.
+
+## What you control
+
+- The agent (any framework, any model, any backend).
+- The judge (LLM, heuristic, ensemble; we don't pick).
+- The mutation strategy (`gepaDriver` for reflective LLM mutation,
+  `evolutionaryDriver({ mutator })` for population search, or
+  implement `ImprovementDriver` directly).
+- The gate (compose `defaultProductionGate` with custom checks via
+  `composeGate`).
+- The deploy step (`autoOnPromote: 'pr'` opens a GitHub PR with the
+  winner; `'none'` returns the surface and you ship however you ship).
+
+## What this does NOT install
+
+- No `@tangle-network/sandbox` — nothing runs in a Tangle sandbox.
+- No hosted orchestrator — traces, artifacts, judge scores stay on
+  your machine (or in `inMemoryCampaignStorage` for Workers/edge).
+- No daemons — `runEval` and `runImprovementLoop` complete in-process
+  and return.
+
+## When you want more
+
+The wedge doc (`docs/design/external-agent-wedge.md`) lays out three
+graduated tiers:
+
+| Tier | What you do | What you get |
+|---|---|---|
+| **LAND** (this quickstart) | `npm i @tangle-network/agent-eval`, wrap dispatch + judge, run loops | Local artifacts; full self-improvement; no Tangle infra |
+| **EXPAND** | Point trace/eval data at our hosted orchestrator | Hosted dashboards, cross-run intelligence, billing on data routed to us |
+| **PLATFORM** | Move execution into our sandbox | Substrate + orchestrator data pre-wired; sandbox usage billing |
+
+Each tier is opt-in. EXPAND and PLATFORM build on the same primitives;
+upgrading is adding configuration, not rewriting your wiring.