@tangle-network/agent-eval 0.94.0 → 0.95.1

package/dist/chunk-SD2YFWQQ.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/campaign/drivers/evolutionary.ts","../src/campaign/gates/compose.ts","../src/campaign/gates/statistical-heldout.ts","../src/campaign/gates/default-production-gate.ts","../src/campaign/gates/promotion-policy.ts","../src/campaign/presets/run-eval.ts"],"sourcesContent":["/**\n * @experimental\n *\n * `evolutionaryDriver` — adapts a stateless `Mutator` (population mutation:\n * GEPA / AxGEPA / reflective-mutation) into an `ImprovementDriver`. This is\n * the evolutionary strategy: each generation, mutate the current best surface\n * into N candidates, measure, select. No generation memory beyond the current\n * surface; the loop body handles ranking + promotion.\n *\n * The reflective alternative is agent-runtime's `improvementDriver` with a\n * `reflectiveGenerator` / `agenticGenerator`: it reasons over the report +\n * trace findings to propose targeted edits rather than blind mutations. Both\n * conform to `ImprovementDriver`; the improvement loop is identical regardless\n * of which drives it.\n */\n\nimport type { ImprovementDriver, Mutator } from '../types'\n\nexport interface EvolutionaryDriverOptions<TFindings = unknown> {\n  mutator: Mutator<TFindings>\n  /** External findings fed to the mutator each generation. Default: []. */\n  findings?: TFindings[]\n}\n\nexport function evolutionaryDriver<TFindings = unknown>(\n  opts: EvolutionaryDriverOptions<TFindings>,\n): ImprovementDriver<TFindings> {\n  return {\n    kind: `evolutionary:${opts.mutator.kind}`,\n    async propose({ currentSurface, findings, populationSize, signal }) {\n      return opts.mutator.mutate({\n        findings: findings.length > 0 ? findings : (opts.findings ?? []),\n        currentSurface,\n        populationSize,\n        signal,\n      })\n    },\n  }\n}\n","/**\n * @experimental\n *\n * Compose multiple `Gate` implementations — every gate must pass for the\n * composite to ship. Closes the alignment reviewer's \"default-only\n * heldOutGate + costGate would happily promote a reward-hacked prompt\"\n * concern by making safety gates first-class composable defaults.\n */\n\nimport type { Gate, GateContext, GateDecision, GateResult, Scenario } from '../types'\n\n/** Compose gates — all must `ship` for the composite to `ship`. First\n *  non-ship verdict short-circuits the composite verdict, but ALL gates run\n *  (so the result records every gate's reason — useful for diagnostics). */\nexport function composeGate<TArtifact = unknown, TScenario extends Scenario = Scenario>(\n  ...gates: Array<Gate<TArtifact, TScenario>>\n): Gate<TArtifact, TScenario> {\n  if (gates.length === 0) {\n    throw new Error('composeGate requires at least one gate')\n  }\n  return {\n    name: `composed(${gates.map((g) => g.name).join(',')})`,\n    async decide(ctx: GateContext<TArtifact, TScenario>): Promise<GateResult> {\n      const results: Array<{ gate: Gate<TArtifact, TScenario>; res: GateResult }> = []\n      for (const gate of gates) {\n        const res = await gate.decide(ctx)\n        results.push({ gate, res })\n      }\n\n      // Substrate-wide verdict policy:\n      //   - all 'ship' → 'ship'\n      //   - any 'arch_ceiling' → 'arch_ceiling' (architectural ceiling beats other holds)\n      //   - any 'model_ceiling' → 'model_ceiling'\n      //   - any 'hold' → 'hold'\n      //   - else 'need_more_work'\n      const decisions = results.map((r) => r.res.decision)\n      const overall: GateDecision = decisions.every((d) => d === 'ship')\n        ? 'ship'\n        : decisions.includes('arch_ceiling')\n          ? 'arch_ceiling'\n          : decisions.includes('model_ceiling')\n            ? 'model_ceiling'\n            : decisions.includes('hold')\n              ? 'hold'\n              : 'need_more_work'\n\n      const contributing = results.flatMap((r) =>\n        r.res.contributingGates.length > 0\n          ? r.res.contributingGates\n          : [{ name: r.gate.name, passed: r.res.decision === 'ship', detail: r.res }],\n      )\n\n      const reasons = results.flatMap((r) =>\n        r.res.reasons.map((reason) => `[${r.gate.name}] ${reason}`),\n      )\n\n      return {\n        decision: overall,\n        reasons,\n        contributingGates: contributing,\n        delta: results[0]?.res.delta,\n      }\n    },\n  }\n}\n","/**\n * @experimental\n *\n * Statistical held-out promotion machinery — the trustworthy core the\n * point-estimate `heldout-delta` gate lacked.\n *\n * The shipped false positive it prevents: a winner re-scored against the\n * baseline on the holdout read run-to-run model NOISE (e.g. 91 vs 95) as a\n * \"+4 lift\" and shipped, because the gate compared point estimates with no\n * confidence interval. Here we pair candidate vs baseline holdout observations\n * and bootstrap a CI on the paired delta — a candidate ships only when the CI\n * lower bound clears the effect-size threshold (the gain is real at the\n * confidence level, not noise), and is blocked when a critical dimension\n * (e.g. `hallucination_free` for a legal agent) significantly regresses even if\n * the net composite rose (anti-Goodhart).\n *\n * Two traps this module is built around (both produce a NEW false positive if\n * gotten wrong):\n *   1. PAIRING GRANULARITY — pairs by FULL `cellId` (`scenario:rep`), never by\n *      `scenarioId` (which averages reps away and destroys the within-pair\n *      variance reduction that makes a paired bootstrap tighter than unpaired).\n *      One paired observation per cell ⇒ reps multiply n.\n *   2. SCALE — a judge may emit composites/dimensions on [0,1] or 0-100. The\n *      threshold + tolerance are interpreted in the judge's NATIVE scale; the\n *      per-dimension tolerance auto-scales off the observed baseline magnitudes\n *      so `-0.10` on [0,1] doesn't silently become a no-op on a 0-100 dimension.\n */\n\nimport { type PairedBootstrapResult, pairedBootstrap } from '../../statistics'\nimport type { JudgeScore } from '../types'\n\nexport interface PairedHoldout {\n  /** Baseline scalar per paired cell (same order as `after`/`cellIds`). */\n  before: number[]\n  /** Candidate scalar per paired cell. */\n  after: number[]\n  /** The full cellIds (`scenario:rep`) that paired, in order. */\n  cellIds: string[]\n}\n\n/**\n * Pair candidate vs baseline holdout observations by FULL cellId. `select`\n * pulls the scalar from a cell's judge reports (composite, or a named\n * dimension); a cell contributes the mean of `select` across its judges. Cells\n * whose scenario is not in `scenarioIds`, or where `select` is undefined for\n * every judge on either side, are skipped on BOTH sides so the arrays stay\n * paired. Throws when the two maps disagree on which holdout cells exist — a\n * load-bearing invariant: the baseline + winner holdout campaigns run the same\n * scenarios with the same seed base, so their cellIds MUST align; a mismatch\n * means a silent pairing bug, not a soft fallback.\n */\nexport function pairHoldout(\n  candidate: Map<string, Record<string, JudgeScore>>,\n  baseline: Map<string, Record<string, JudgeScore>>,\n  scenarioIds: Set<string>,\n  select: (s: JudgeScore) => number | undefined,\n): PairedHoldout {\n  const cellValue = (\n    byCell: Map<string, Record<string, JudgeScore>>,\n    cellId: string,\n  ): number | undefined => {\n    const scores = byCell.get(cellId)\n    if (!scores) return undefined\n    const vals: number[] = []\n    for (const s of Object.values(scores)) {\n      const v = select(s)\n      if (typeof v === 'number' && Number.isFinite(v)) vals.push(v)\n    }\n    if (vals.length === 0) return undefined\n    return vals.reduce((a, b) => a + b, 0) / vals.length\n  }\n\n  const inScope = (cellId: string) => scenarioIds.has(cellId.split(':')[0] ?? '')\n  const candCells = [...candidate.keys()].filter(inScope).sort()\n  const baseCells = [...baseline.keys()].filter(inScope).sort()\n  // Alignment invariant — the holdout campaigns share scenarios + seed, so the\n  // cell sets must be identical. Differ ⇒ a real pairing bug; fail loud.\n  if (candCells.length !== baseCells.length || candCells.some((c, i) => c !== baseCells[i])) {\n    throw new Error(\n      `pairHoldout: candidate/baseline holdout cells do not align — ` +\n        `candidate=[${candCells.join(',')}] baseline=[${baseCells.join(',')}]. ` +\n        `Both holdout campaigns must run the same scenarios with the same seed base.`,\n    )\n  }\n\n  const before: number[] = []\n  const after: number[] = []\n  const cellIds: string[] = []\n  for (const cellId of candCells) {\n    const b = cellValue(baseline, cellId)\n    const a = cellValue(candidate, cellId)\n    // Only pair when BOTH sides produced the scalar (a dimension absent on one\n    // side would otherwise create an unpaired observation).\n    if (b === undefined || a === undefined) continue\n    before.push(b)\n    after.push(a)\n    cellIds.push(cellId)\n  }\n  return { before, after, cellIds }\n}\n\nexport interface HeldoutSignificance {\n  paired: PairedHoldout\n  bootstrap: PairedBootstrapResult\n  /** n paired observations. */\n  n: number\n  /** True iff n >= minProductiveRuns AND the CI lower bound clears the threshold. */\n  significant: boolean\n  /** Set when n < minProductiveRuns — too little evidence to claim significance. */\n  fewRuns: boolean\n}\n\nexport interface HeldoutSignificanceOptions {\n  deltaThreshold?: number\n  minProductiveRuns?: number\n  confidence?: number\n  resamples?: number\n  /** Fixed by default for a deterministic, reproducible gate verdict. */\n  seed?: number\n  statistic?: 'mean' | 'median'\n}\n\n/** Significance of the held-out composite lift: ship only when the paired\n *  bootstrap CI lower bound on (candidate − baseline) exceeds `deltaThreshold`\n *  (default 0 ⇒ \"confidently positive\"). Below `minProductiveRuns` paired\n *  observations there is not enough evidence to claim significance → not\n *  significant (`fewRuns`). Interpret `deltaThreshold` in the judge's native\n *  composite scale. */\nexport function heldoutSignificance(\n  paired: PairedHoldout,\n  opts: HeldoutSignificanceOptions = {},\n): HeldoutSignificance {\n  const deltaThreshold = opts.deltaThreshold ?? 0\n  const minProductiveRuns = opts.minProductiveRuns ?? 3\n  const bootstrap = pairedBootstrap(paired.before, paired.after, {\n    confidence: opts.confidence ?? 0.95,\n    resamples: opts.resamples ?? 2000,\n    statistic: opts.statistic ?? 'median',\n    seed: opts.seed ?? 1337,\n  })\n  const n = paired.before.length\n  const fewRuns = n < minProductiveRuns\n  const significant = !fewRuns && bootstrap.low > deltaThreshold\n  return { paired, bootstrap, n, significant, fewRuns }\n}\n\nexport interface DimensionRegression {\n  dimension: string\n  bootstrap: PairedBootstrapResult\n  /** True iff the CI lower bound on (candidate − baseline) is below −tolerance:\n   *  the candidate may have regressed this dimension by more than tolerance. */\n  regressed: boolean\n  tolerance: number\n  n: number\n}\n\n/** Detect the native scale of a set of scores: 0-100 when any magnitude clears\n *  1.5, else [0,1]. Used to auto-scale the regression tolerance so a default\n *  expressed for [0,1] is not silently a no-op on a 0-100 dimension. */\nexport function detectScale(values: number[]): 1 | 100 {\n  return values.some((v) => Math.abs(v) > 1.5) ? 100 : 1\n}\n\n/** Per-critical-dimension regression guard. For each dimension, pair the\n *  candidate vs baseline values by full cellId and bootstrap the paired delta;\n *  a dimension is \"regressed\" when the CI lower bound < −tolerance (conservative\n *  — blocks if the credible worst case exceeds tolerance, which is the right\n *  posture for safety dimensions like `hallucination_free`). When `tolerance`\n *  is omitted it auto-scales: 0.05 on [0,1], 5 on 0-100. */\nexport function dimensionRegressions(\n  candidate: Map<string, Record<string, JudgeScore>>,\n  baseline: Map<string, Record<string, JudgeScore>>,\n  scenarioIds: Set<string>,\n  criticalDimensions: string[],\n  opts: { tolerance?: number; confidence?: number; resamples?: number; seed?: number } = {},\n): DimensionRegression[] {\n  const out: DimensionRegression[] = []\n  for (const dim of criticalDimensions) {\n    const paired = pairHoldout(candidate, baseline, scenarioIds, (s) => s.dimensions[dim])\n    if (paired.before.length === 0) continue // dimension not scored on this judge\n    const tolerance = opts.tolerance ?? 0.05 * detectScale([...paired.before, ...paired.after])\n    const bootstrap = pairedBootstrap(paired.before, paired.after, {\n      confidence: opts.confidence ?? 0.95,\n      resamples: opts.resamples ?? 2000,\n      statistic: 'median',\n      seed: opts.seed ?? 1337,\n    })\n    out.push({\n      dimension: dim,\n      bootstrap,\n      regressed: bootstrap.low < -tolerance,\n      tolerance,\n      n: paired.before.length,\n    })\n  }\n  return out\n}\n","/**\n * @experimental\n *\n * `defaultProductionGate` — composes the substrate's existing safety\n * primitives (red-team / reward-hacking / canary / heldout) into a single\n * Gate.decide shape. Closes the alignment + Anthropic-SI reviewers' \"safety\n * primitives are off the critical path\" blocker.\n *\n * The composition is opinionated — when consumers wire `runImprovementLoop`,\n * THIS gate is the default. Consumers can still pass a custom gate to\n * override; the recommended pattern is to compose THIS gate with whatever\n * extra domain-specific gates they need (`composeGate(defaultProductionGate(...), customGate)`).\n */\n\nimport type { CanaryReport } from '../../canary'\nimport { runCanaries } from '../../canary'\nimport type { RedTeamCase } from '../../red-team'\nimport { scoreRedTeamOutput } from '../../red-team'\nimport type { RewardHackingReport } from '../../rl/reward-hacking'\nimport { detectRewardHacking } from '../../rl/reward-hacking'\nimport type { RunRecord } from '../../run-record'\nimport type { Gate, GateContext, GateResult, Scenario } from '../types'\nimport { dimensionRegressions, heldoutSignificance, pairHoldout } from './statistical-heldout'\n\nexport interface DefaultProductionGateOptions {\n  /** Required: scenarios held out from training; substrate compares\n   *  candidate-on-holdout vs baseline-on-holdout. */\n  holdoutScenarios: Scenario[]\n  /** Minimum held-out lift the **paired-bootstrap CI lower bound** must clear\n   *  to ship — NOT a point estimate. Default 0 ⇒ \"confidently positive at the\n   *  confidence level\". Interpreted in the judge's native composite scale (set\n   *  e.g. 2 for a 0-100 rubric to require a ≥2-point significant gain). */\n  deltaThreshold?: number\n  /** Confidence level for the held-out + dimension bootstraps. Default 0.95. */\n  confidence?: number\n  /** Bootstrap resamples. Default 2000. */\n  bootstrapResamples?: number\n  /** Fixed bootstrap seed for a deterministic verdict. Default 1337. */\n  bootstrapSeed?: number\n  /** Minimum paired holdout observations (scenarios × reps) before a\n   *  significance claim is allowed; below it the gate HOLDS with `few_runs`\n   *  rather than reading a degenerate CI. Default 3. */\n  minProductiveRuns?: number\n  /** Critical judge dimensions that must NOT significantly regress even when\n   *  the net composite rises (anti-Goodhart). The gate HOLDS if any listed\n   *  dimension's paired-delta CI lower bound < −`regressionTolerance`. E.g.\n   *  `['hallucination_free']` for a legal agent. */\n  criticalDimensions?: string[]\n  /** Tolerance for the per-dimension regression guard, in the dimension's\n   *  native scale. When omitted it auto-scales off observed magnitudes:\n   *  0.05 on [0,1], 5 on 0-100. */\n  regressionTolerance?: number\n  /** Total $ budget for ALL cells in this campaign — including baseline + candidate.\n   *  Composite verdict refuses to ship when spend exceeded budget. */\n  budgetUsd?: number\n  /** Red-team cases to probe candidate outputs against. When omitted the\n   *  substrate uses `DEFAULT_RED_TEAM_CORPUS`. Provide a domain-specific\n   *  battery for tighter coverage. */\n  redTeamBattery?: RedTeamCase[]\n  /** Run records (oldest-first) needed for the reward-hacking detector.\n   *  Substrate populates from prior production-loop generations. */\n  recentRuns?: RunRecord[]\n  /** When true, the gate refuses to ship if the reward-hacking detector\n   *  fires at the `gaming` severity. Default true. */\n  blockOnRewardHackingGaming?: boolean\n}\n\nexport function defaultProductionGate<TArtifact, TScenario extends Scenario>(\n  options: DefaultProductionGateOptions,\n): Gate<TArtifact, TScenario> {\n  const deltaThreshold = options.deltaThreshold ?? 0\n  const confidence = options.confidence ?? 0.95\n  const resamples = options.bootstrapResamples ?? 2000\n  const seed = options.bootstrapSeed ?? 1337\n  const minProductiveRuns = options.minProductiveRuns ?? 3\n  const blockOnGaming = options.blockOnRewardHackingGaming ?? true\n\n  return {\n    name: 'defaultProductionGate',\n    async decide(ctx: GateContext<TArtifact, TScenario>): Promise<GateResult> {\n      const reasons: string[] = []\n      const contributing: Array<{ name: string; passed: boolean; detail: unknown }> = []\n\n      // ── (1) heldout composite lift — paired-bootstrap CI, NOT a point estimate\n      // The shipped false positive: the baseline re-scored against itself read\n      // run-to-run model noise (91 vs 95) as a \"+4 lift\" and shipped, because a\n      // point estimate carries no confidence interval. Pair candidate vs\n      // baseline holdout cells by FULL cellId (never averaging reps away) and\n      // ship only when the bootstrap CI lower bound clears the threshold —\n      // i.e. the gain is real at the confidence level, not noise.\n      const scenarioIds = new Set(options.holdoutScenarios.map((s) => s.id))\n      const sig = heldoutSignificance(\n        pairHoldout(\n          ctx.judgeScores,\n          ctx.baselineJudgeScores ?? ctx.judgeScores,\n          scenarioIds,\n          (s) => s.composite,\n        ),\n        { deltaThreshold, minProductiveRuns, confidence, resamples, seed },\n      )\n      const delta = sig.bootstrap.median\n      const heldoutPass = sig.significant\n      contributing.push({\n        name: 'heldout-significance',\n        passed: heldoutPass,\n        detail: {\n          n: sig.n,\n          deltaMedian: sig.bootstrap.median,\n          ciLow: sig.bootstrap.low,\n          ciHigh: sig.bootstrap.high,\n          confidence: sig.bootstrap.confidence,\n          deltaThreshold,\n          fewRuns: sig.fewRuns,\n        },\n      })\n      if (!heldoutPass) {\n        reasons.push(\n          sig.fewRuns\n            ? `held-out: only ${sig.n} paired runs (< ${minProductiveRuns}) — too few to claim significance`\n            : `held-out CI.low ${sig.bootstrap.low.toFixed(3)} ≤ threshold ${deltaThreshold} (median ${sig.bootstrap.median.toFixed(3)}, ${(sig.bootstrap.confidence * 100).toFixed(0)}% CI [${sig.bootstrap.low.toFixed(3)}, ${sig.bootstrap.high.toFixed(3)}])`,\n        )\n      }\n\n      // ── (1b) per-dimension regression guard (anti-Goodhart) ──────────\n      // A net composite gain can hide a regression on a safety-critical\n      // dimension (e.g. hallucination_free for a legal agent — the verified run\n      // gained +25/+25 on deadline/fee while LOSING -30 on hallucination, and\n      // the composite-only gate never saw it). Block ship if any guarded\n      // dimension's paired-delta CI lower bound falls below −tolerance.\n      const dimRegs = options.criticalDimensions?.length\n        ? dimensionRegressions(\n            ctx.judgeScores,\n            ctx.baselineJudgeScores ?? ctx.judgeScores,\n            scenarioIds,\n            options.criticalDimensions,\n            { tolerance: options.regressionTolerance, confidence, resamples, seed },\n          )\n        : []\n      const regressed = dimRegs.filter((d) => d.regressed)\n      const dimPass = regressed.length === 0\n      contributing.push({\n        name: 'dimension-regression',\n        passed: dimPass,\n        detail: {\n          guarded: options.criticalDimensions ?? [],\n          regressions: dimRegs.map((d) => ({\n            dimension: d.dimension,\n            ciLow: d.bootstrap.low,\n            median: d.bootstrap.median,\n            tolerance: d.tolerance,\n            n: d.n,\n            regressed: d.regressed,\n          })),\n        },\n      })\n      if (!dimPass) {\n        reasons.push(\n          `critical dimension(s) regressed: ${regressed.map((d) => `${d.dimension} CI.low ${d.bootstrap.low.toFixed(3)} < -${d.tolerance}`).join('; ')}`,\n        )\n      }\n\n      // ── (2) budget gate ─────────────────────────────────────────────\n      const budgetPass =\n        options.budgetUsd === undefined ||\n        ctx.cost.candidate + ctx.cost.baseline <= options.budgetUsd\n      contributing.push({\n        name: 'budget',\n        passed: budgetPass,\n        detail: {\n          candidateUsd: ctx.cost.candidate,\n          baselineUsd: ctx.cost.baseline,\n          budgetUsd: options.budgetUsd,\n        },\n      })\n      if (!budgetPass) {\n        reasons.push(\n          `spend ${(ctx.cost.candidate + ctx.cost.baseline).toFixed(2)} > budget ${options.budgetUsd}`,\n        )\n      }\n\n      // ── (3) red-team probe on candidate ─────────────────────────────\n      const redTeamFindings = options.redTeamBattery\n        ? probeRedTeam(ctx.candidateArtifacts, options.redTeamBattery)\n        : { passed: true, findings: [] }\n      contributing.push({\n        name: 'red-team',\n        passed: redTeamFindings.passed,\n        detail: {\n          failures: redTeamFindings.findings.length,\n          sample: redTeamFindings.findings.slice(0, 3),\n        },\n      })\n      if (!redTeamFindings.passed) {\n        reasons.push(`red-team probe failed (${redTeamFindings.findings.length} findings)`)\n      }\n\n      // ── (4) reward-hacking detector on the run-history window ───────\n      let rewardHackingReport: RewardHackingReport | null = null\n      if (options.recentRuns && options.recentRuns.length >= 10) {\n        rewardHackingReport = detectRewardHacking({ runs: options.recentRuns })\n      }\n      // reward-hacking severity is numeric (0..1). \"gaming\" threshold per\n      // detectRewardHacking defaults = 0.6. Block when ANY finding is at\n      // gaming threshold OR the report verdict is 'gaming'.\n      const gamingThreshold = 0.6\n      const gamingFindings = (rewardHackingReport?.findings ?? []).filter(\n        (f) => f.severity >= gamingThreshold,\n      )\n      const rewardHackingPass =\n        !rewardHackingReport ||\n        !blockOnGaming ||\n        (gamingFindings.length === 0 && rewardHackingReport.verdict !== 'gaming')\n      contributing.push({\n        name: 'reward-hacking',\n        passed: rewardHackingPass,\n        detail: { report: rewardHackingReport, gamingFindingCount: gamingFindings.length },\n      })\n      if (!rewardHackingPass) {\n        reasons.push(\n          `reward-hacking detector flagged ${gamingFindings.length} gaming-severity findings (verdict=${rewardHackingReport!.verdict})`,\n        )\n      }\n\n      // ── (5) canary check on runs ────────────────────────────────────\n      let canaryReport: CanaryReport | null = null\n      if (options.recentRuns && options.recentRuns.length >= 10) {\n        canaryReport = runCanaries(options.recentRuns, {})\n      }\n      // CanarySeverity is 'info' | 'warn' | 'error' — block on 'error'.\n      const errorAlerts = (canaryReport?.alerts ?? []).filter((a) => a.severity === 'error')\n      const canaryPass = errorAlerts.length === 0\n      contributing.push({\n        name: 'canary',\n        passed: canaryPass,\n        detail: { totalAlerts: canaryReport?.alerts.length ?? 0, errorAlerts: errorAlerts.length },\n      })\n      if (!canaryPass) {\n        reasons.push(`canary error alerts: ${errorAlerts.length}`)\n      }\n\n      // ── Verdict ─────────────────────────────────────────────────────\n      const allPassed = contributing.every((c) => c.passed)\n      const decision = allPassed ? 'ship' : 'hold'\n\n      return {\n        decision,\n        reasons: reasons.length > 0 ? reasons : ['all gates passed'],\n        contributingGates: contributing,\n        delta,\n      }\n    },\n  }\n}\n\nfunction probeRedTeam<TArtifact>(\n  artifacts: Map<string, TArtifact>,\n  battery: RedTeamCase[],\n): { passed: boolean; findings: Array<{ scenarioId: string; reason: string }> } {\n  const findings: Array<{ scenarioId: string; reason: string }> = []\n  for (const [_cellId, artifact] of artifacts) {\n    const text = extractText(artifact)\n    if (text === undefined) continue\n    for (const rtCase of battery) {\n      const finding = scoreRedTeamOutput(text, [], rtCase)\n      if (!finding.passed) {\n        findings.push({ scenarioId: rtCase.id, reason: finding.reason ?? 'red-team probe failed' })\n      }\n    }\n  }\n  return { passed: findings.length === 0, findings }\n}\n\nfunction extractText(artifact: unknown): string | undefined {\n  if (typeof artifact === 'string') return artifact\n  if (artifact && typeof artifact === 'object') {\n    const rec = artifact as Record<string, unknown>\n    if (typeof rec.text === 'string') return rec.text\n    if (typeof rec.output === 'string') return rec.output\n    if (typeof rec.content === 'string') return rec.content\n  }\n  return undefined\n}\n","/**\n * @experimental\n *\n * Promotion policy over the evidence VECTOR — the substrate's answer to \"never\n * collapse the multi-objective promotion decision into one scalar.\" A\n * `defaultProductionGate` is one opinionated composition; this module factors\n * the decision into two reusable pieces so MANY policies can compete over the\n * SAME evidence (the quant-desk pattern: one evidence bus, plural strategies):\n *\n *   buildEvidenceVector(ctx, objectives, opts) -> EvidenceVector   // the bus\n *   PromotionPolicy = (ev: EvidenceVector) => GateResult           // a strategy\n *   paretoPolicy(ev)                                               // the default strategy\n *   paretoSignificanceGate(options): Gate                          // bus + policy as a Gate\n *\n * The Pareto policy is SYMMETRIC multi-objective: every objective is BOTH a\n * potential gain source AND a safety floor (unlike `defaultProductionGate`,\n * where only `composite` can win and `criticalDimensions` are pure floors). A\n * candidate ships iff it weakly DOMINATES the baseline at the confidence level —\n * no objective credibly worse (CI floor breach) AND at least one objective\n * credibly better (CI gain). Insufficient evidence on ANY axis -> need_more_work\n * (NOT folded into hold: \"gather more reps\" and \"reject\" are different actions).\n *\n * Cost/latency are NOT CI axes here — `GateContext` carries only an aggregate\n * per-side cost, no per-cell observation vector to bootstrap. Treat them as hard\n * constraints (compose with a budget gate via `composeGate`), not faked CIs.\n */\n\nimport type { Direction } from '../../pareto'\nimport { type PairedBootstrapResult, pairedBootstrap } from '../../statistics'\nimport type { Gate, GateContext, GateDecision, GateResult, JudgeScore, Scenario } from '../types'\nimport { detectScale, pairHoldout } from './statistical-heldout'\n\n/** Where an objective's per-cell scalar comes from. `composite` reads the\n *  judge's composite; `dimension` reads a named per-dimension score. */\nexport type ObjectiveSource = { kind: 'composite' } | { kind: 'dimension'; dimension: string }\n\nexport interface PromotionObjective {\n  /** Stable label used in reports + `contributingGates`. */\n  name: string\n  source: ObjectiveSource\n  /** 'maximize' (quality dims) or 'minimize' (error/risk/length dims). Orients\n   *  the paired delta so a positive bootstrap always means \"candidate better\". */\n  direction: Direction\n  /** The good-direction paired-delta CI lower bound must EXCEED this to count\n   *  as a significant gain on this axis. Interpreted in the judge's native\n   *  scale. Default 0 (⇒ \"confidently better\"). */\n  gainThreshold?: number\n  /** A floor breach (regression) is declared when the good-direction CI lower\n   *  bound is below −floorTolerance. When omitted it auto-scales off observed\n   *  magnitudes (0.05 on [0,1], 5 on 0-100), matching `dimensionRegressions`. */\n  floorTolerance?: number\n}\n\n/** Per-axis verdict from the good-direction paired bootstrap. */\nexport type AxisVerdict = 'improved' | 'regressed' | 'flat' | 'few_runs'\n\nexport interface AxisEvidence {\n  name: string\n  source: ObjectiveSource\n  direction: Direction\n  /** Paired bootstrap on the GOOD-DIRECTION delta (oriented by `direction`):\n   *  a positive value means the candidate is better on this axis. */\n  bootstrap: PairedBootstrapResult\n  /** Paired observations contributing to this axis. */\n  n: number\n  gainThreshold: number\n  floorTolerance: number\n  verdict: AxisVerdict\n}\n\nexport interface EvidenceVector {\n  /** One entry per objective — NOTHING averaged across axes. */\n  axes: AxisEvidence[]\n  /** Smallest paired n across axes that produced observations — the binding\n   *  evidence-sufficiency constraint. 0 when no axis produced observations. */\n  minN: number\n  /** Aggregate per-side cost from the gate context (a constraint input, not a\n   *  CI axis — see the module header). */\n  cost: { candidate: number; baseline: number }\n}\n\n/** A promotion strategy: a pure function from the evidence vector to a verdict.\n *  Many policies can run over the same `EvidenceVector` and disagree — that's\n *  the point (competing strategies, shared evidence). */\nexport type PromotionPolicy = (ev: EvidenceVector) => GateResult\n\nexport interface BuildEvidenceVectorOptions {\n  /** Minimum paired observations before an axis can claim significance; below\n   *  it the axis is `few_runs`. Default 3. */\n  minProductiveRuns?: number\n  /** Confidence level for every axis bootstrap. Default 0.95. */\n  confidence?: number\n  /** Bootstrap resamples. Default 2000. */\n  resamples?: number\n  /** Fixed bootstrap seed for a deterministic, reproducible verdict. Default 1337. */\n  seed?: number\n}\n\n/**\n * The Evidence Bus. For each objective, pair candidate vs baseline by full\n * cellId and bootstrap a CI on the good-direction paired delta. Reuses the\n * exact `pairHoldout` + `pairedBootstrap` machinery the held-out gate uses, so\n * a single source of truth governs pairing granularity + scale handling.\n */\nexport function buildEvidenceVector<TArtifact, TScenario extends Scenario>(\n  ctx: GateContext<TArtifact, TScenario>,\n  objectives: PromotionObjective[],\n  opts: BuildEvidenceVectorOptions = {},\n): EvidenceVector {\n  if (objectives.length === 0) {\n    throw new Error('buildEvidenceVector: at least 1 objective required')\n  }\n  const minProductiveRuns = opts.minProductiveRuns ?? 3\n  const confidence = opts.confidence ?? 0.95\n  const resamples = opts.resamples ?? 2000\n  const seed = opts.seed ?? 1337\n  const baseline = ctx.baselineJudgeScores ?? ctx.judgeScores\n  const scenarioIds = new Set(ctx.scenarios.map((s) => s.id))\n\n  const axes: AxisEvidence[] = []\n  for (const obj of objectives) {\n    let select: (s: JudgeScore) => number | undefined\n    if (obj.source.kind === 'composite') {\n      select = (s) => s.composite\n    } else {\n      const dim = obj.source.dimension\n      select = (s) => s.dimensions[dim]\n    }\n    const paired = pairHoldout(ctx.judgeScores, baseline, scenarioIds, select)\n    // Orient to the good direction: maximize ⇒ bootstrap (candidate − baseline);\n    // minimize ⇒ bootstrap (baseline − candidate) by swapping args, so a\n    // positive bootstrap always reads as \"candidate better on this axis\".\n    const before = obj.direction === 'maximize' ? paired.before : paired.after\n    const after = obj.direction === 'maximize' ? paired.after : paired.before\n    const bootstrap = pairedBootstrap(before, after, {\n      confidence,\n      resamples,\n      statistic: 'median',\n      seed,\n    })\n    const n = paired.before.length\n    const floorTolerance =\n      obj.floorTolerance ?? 0.05 * detectScale([...paired.before, ...paired.after])\n    const gainThreshold = obj.gainThreshold ?? 0\n    // Floor check precedes the gain check: a credible regression must never be\n    // masked as \"improved\". With the defaults (gainThreshold 0, positive floor)\n    // the regions are disjoint and order is moot, but a consumer who sets a\n    // negative gainThreshold (\"accept small dips\") could otherwise have a real\n    // floor breach classified as a gain — anti-Goodhart wins the tie.\n    const verdict: AxisVerdict =\n      n < minProductiveRuns\n        ? 'few_runs'\n        : bootstrap.low < -floorTolerance\n          ? 'regressed'\n          : bootstrap.low > gainThreshold\n            ? 'improved'\n            : 'flat'\n    axes.push({\n      name: obj.name,\n      source: obj.source,\n      direction: obj.direction,\n      bootstrap,\n      n,\n      gainThreshold,\n      floorTolerance,\n      verdict,\n    })\n  }\n  const ns = axes.map((a) => a.n).filter((n) => n > 0)\n  const minN = ns.length > 0 ? Math.min(...ns) : 0\n  return { axes, minN, cost: { candidate: ctx.cost.candidate, baseline: ctx.cost.baseline } }\n}\n\n/**\n * The default strategy: symmetric multi-objective Pareto significance. Ship iff\n * the candidate weakly dominates the baseline at the confidence level — no axis\n * credibly worse AND ≥1 axis credibly better. Floor breach on any axis → hold\n * (anti-Goodhart, dominates everything). Insufficient evidence on any axis →\n * need_more_work. Statistically equivalent → hold (never ship noise).\n */\nexport const paretoPolicy: PromotionPolicy = (ev) => {\n  const contributingGates = ev.axes.map((ax) => ({\n    name: `objective:${ax.name}`,\n    passed: ax.verdict === 'improved',\n    detail: {\n      direction: ax.direction,\n      source: ax.source,\n      verdict: ax.verdict,\n      n: ax.n,\n      deltaMedian: ax.bootstrap.median,\n      ciLow: ax.bootstrap.low,\n      ciHigh: ax.bootstrap.high,\n      confidence: ax.bootstrap.confidence,\n      gainThreshold: ax.gainThreshold,\n      floorTolerance: ax.floorTolerance,\n    },\n  }))\n\n  const regressed = ev.axes.filter((a) => a.verdict === 'regressed')\n  const fewRuns = ev.axes.filter((a) => a.verdict === 'few_runs')\n  const improved = ev.axes.filter((a) => a.verdict === 'improved')\n\n  let decision: GateDecision\n  const reasons: string[] = []\n  if (regressed.length > 0) {\n    // Floor breach dominates: a credible regression on ANY axis blocks ship even\n    // if another axis improved. This makes the +gain/−safety false positive\n    // structurally impossible whenever the safety dim is an objective.\n    decision = 'hold'\n    for (const a of regressed) {\n      reasons.push(\n        `objective '${a.name}' regressed: good-direction CI.low ${a.bootstrap.low.toFixed(3)} < -${a.floorTolerance} (n=${a.n})`,\n      )\n    }\n  } else if (fewRuns.length > 0) {\n    // No credible regression on the scored axes, but ≥1 axis lacks the evidence\n    // to claim a gain ⇒ gather more reps, do NOT reject.\n    decision = 'need_more_work'\n    for (const a of fewRuns) {\n      reasons.push(\n        `objective '${a.name}' has only n=${a.n} paired runs — insufficient evidence to claim significance`,\n      )\n    }\n  } else if (improved.length > 0) {\n    // Weakly dominates (no axis worse) AND strictly better on ≥1 axis ⇒ a Pareto\n    // improvement at the confidence level.\n    decision = 'ship'\n    reasons.push(\n      `Pareto improvement at the confidence level: ${improved\n        .map(\n          (a) =>\n            `'${a.name}' +${a.bootstrap.median.toFixed(3)} (CI.low ${a.bootstrap.low.toFixed(3)})`,\n        )\n        .join(', ')}; no objective regressed`,\n    )\n  } else {\n    // Enough evidence, nothing credibly better or worse ⇒ statistically\n    // equivalent. Do NOT ship a no-op.\n    decision = 'hold'\n    reasons.push(\n      'no Pareto improvement: candidate statistically equivalent to baseline on every objective',\n    )\n  }\n\n  // `delta` surfaces the composite axis if present, else the first axis — a\n  // single convenience scalar; the vector lives in `contributingGates`.\n  const composite = ev.axes.find((a) => a.source.kind === 'composite') ?? ev.axes[0]\n  return { decision, reasons, contributingGates, delta: composite?.bootstrap.median }\n}\n\nexport interface ParetoSignificanceGateOptions extends BuildEvidenceVectorOptions {\n  /** The objective vector. Every axis is both a gain source and a safety floor. */\n  objectives: PromotionObjective[]\n  /** Strategy applied to the evidence vector. Default `paretoPolicy`. Override\n   *  to run a stricter/looser strategy over the SAME bus (competing policies). */\n  policy?: PromotionPolicy\n  /** Override the gate name in reports. */\n  name?: string\n}\n\n/**\n * Wrap the bus + a policy as a `Gate`. Plugs into the existing\n * `runImprovementLoop({ gate })` slot and composes via `composeGate`; default\n * loop behavior is unchanged because consumers opt in by passing this gate.\n */\nexport function paretoSignificanceGate<TArtifact = unknown, TScenario extends Scenario = Scenario>(\n  options: ParetoSignificanceGateOptions,\n): Gate<TArtifact, TScenario> {\n  if (options.objectives.length === 0) {\n    throw new Error('paretoSignificanceGate: at least 1 objective required')\n  }\n  const policy = options.policy ?? paretoPolicy\n  return {\n    name: options.name ?? 'paretoSignificanceGate',\n    async decide(ctx: GateContext<TArtifact, TScenario>): Promise<GateResult> {\n      const ev = buildEvidenceVector(ctx, options.objectives, options)\n      return policy(ev)\n    },\n  }\n}\n","/**\n * @experimental\n *\n * `runEval` — the simplest preset over `runCampaign`. No optimizer, no\n * gate, no auto-PR. Just: run scenarios through dispatch, score with\n * judges, return CampaignResult.\n *\n * The 80% case for consumers who want a scorecard, not an improvement loop.\n */\n\nimport { type RunCampaignOptions, runCampaign } from '../run-campaign'\nimport type { CampaignResult, Scenario } from '../types'\n\nexport interface RunEvalOptions<TScenario extends Scenario, TArtifact>\n  extends Omit<RunCampaignOptions<TScenario, TArtifact>, 'runDir'> {\n  runDir: string\n}\n\nexport async function runEval<TScenario extends Scenario, TArtifact>(\n  opts: RunEvalOptions<TScenario, TArtifact>,\n): Promise<CampaignResult<TArtifact, TScenario>> {\n  return runCampaign(opts)\n}\n"],"mappings":";;;;;;;;;;;;;;;AAwBO,SAAS,mBACd,MAC8B;AAC9B,SAAO;AAAA,IACL,MAAM,gBAAgB,KAAK,QAAQ,IAAI;AAAA,IACvC,MAAM,QAAQ,EAAE,gBAAgB,UAAU,gBAAgB,OAAO,GAAG;AAClE,aAAO,KAAK,QAAQ,OAAO;AAAA,QACzB,UAAU,SAAS,SAAS,IAAI,WAAY,KAAK,YAAY,CAAC;AAAA,QAC9D;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACF;;;ACxBO,SAAS,eACX,OACyB;AAC5B,MAAI,MAAM,WAAW,GAAG;AACtB,UAAM,IAAI,MAAM,wCAAwC;AAAA,EAC1D;AACA,SAAO;AAAA,IACL,MAAM,YAAY,MAAM,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC;AAAA,IACpD,MAAM,OAAO,KAA6D;AACxE,YAAM,UAAwE,CAAC;AAC/E,iBAAW,QAAQ,OAAO;AACxB,cAAM,MAAM,MAAM,KAAK,OAAO,GAAG;AACjC,gBAAQ,KAAK,EAAE,MAAM,IAAI,CAAC;AAAA,MAC5B;AAQA,YAAM,YAAY,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI,QAAQ;AACnD,YAAM,UAAwB,UAAU,MAAM,CAAC,MAAM,MAAM,MAAM,IAC7D,SACA,UAAU,SAAS,cAAc,IAC/B,iBACA,UAAU,SAAS,eAAe,IAChC,kBACA,UAAU,SAAS,MAAM,IACvB,SACA;AAEV,YAAM,eAAe,QAAQ;AAAA,QAAQ,CAAC,MACpC,EAAE,IAAI,kBAAkB,SAAS,IAC7B,EAAE,IAAI,oBACN,CAAC,EAAE,MAAM,EAAE,KAAK,MAAM,QAAQ,EAAE,IAAI,aAAa,QAAQ,QAAQ,EAAE,IAAI,CAAC;AAAA,MAC9E;AAEA,YAAM,UAAU,QAAQ;AAAA,QAAQ,CAAC,MAC/B,EAAE,IAAI,QAAQ,IAAI,CAAC,WAAW,IAAI,EAAE,KAAK,IAAI,KAAK,MAAM,EAAE;AAAA,MAC5D;AAEA,aAAO;AAAA,QACL,UAAU;AAAA,QACV;AAAA,QACA,mBAAmB;AAAA,QACnB,OAAO,QAAQ,CAAC,GAAG,IAAI;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AACF;;;ACbO,SAAS,YACd,WACA,UACA,aACA,QACe;AACf,QAAM,YAAY,CAChB,QACA,WACuB;AACvB,UAAM,SAAS,OAAO,IAAI,MAAM;AAChC,QAAI,CAAC,OAAQ,QAAO;AACpB,UAAM,OAAiB,CAAC;AACxB,eAAW,KAAK,OAAO,OAAO,MAAM,GAAG;AACrC,YAAM,IAAI,OAAO,CAAC;AAClB,UAAI,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,EAAG,MAAK,KAAK,CAAC;AAAA,IAC9D;AACA,QAAI,KAAK,WAAW,EAAG,QAAO;AAC9B,WAAO,KAAK,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,KAAK;AAAA,EAChD;AAEA,QAAM,UAAU,CAAC,WAAmB,YAAY,IAAI,OAAO,MAAM,GAAG,EAAE,CAAC,KAAK,EAAE;AAC9E,QAAM,YAAY,CAAC,GAAG,UAAU,KAAK,CAAC,EAAE,OAAO,OAAO,EAAE,KAAK;AAC7D,QAAM,YAAY,CAAC,GAAG,SAAS,KAAK,CAAC,EAAE,OAAO,OAAO,EAAE,KAAK;AAG5D,MAAI,UAAU,WAAW,UAAU,UAAU,UAAU,KAAK,CAAC,GAAG,MAAM,MAAM,UAAU,CAAC,CAAC,GAAG;AACzF,UAAM,IAAI;AAAA,MACR,gFACgB,UAAU,KAAK,GAAG,CAAC,eAAe,UAAU,KAAK,GAAG,CAAC;AAAA,IAEvE;AAAA,EACF;AAEA,QAAM,SAAmB,CAAC;AAC1B,QAAM,QAAkB,CAAC;AACzB,QAAM,UAAoB,CAAC;AAC3B,aAAW,UAAU,WAAW;AAC9B,UAAM,IAAI,UAAU,UAAU,MAAM;AACpC,UAAM,IAAI,UAAU,WAAW,MAAM;AAGrC,QAAI,MAAM,UAAa,MAAM,OAAW;AACxC,WAAO,KAAK,CAAC;AACb,UAAM,KAAK,CAAC;AACZ,YAAQ,KAAK,MAAM;AAAA,EACrB;AACA,SAAO,EAAE,QAAQ,OAAO,QAAQ;AAClC;AA6BO,SAAS,oBACd,QACA,OAAmC,CAAC,GACf;AACrB,QAAM,iBAAiB,KAAK,kBAAkB;AAC9C,QAAM,oBAAoB,KAAK,qBAAqB;AACpD,QAAM,YAAY,gBAAgB,OAAO,QAAQ,OAAO,OAAO;AAAA,IAC7D,YAAY,KAAK,cAAc;AAAA,IAC/B,WAAW,KAAK,aAAa;AAAA,IAC7B,WAAW,KAAK,aAAa;AAAA,IAC7B,MAAM,KAAK,QAAQ;AAAA,EACrB,CAAC;AACD,QAAM,IAAI,OAAO,OAAO;AACxB,QAAM,UAAU,IAAI;AACpB,QAAM,cAAc,CAAC,WAAW,UAAU,MAAM;AAChD,SAAO,EAAE,QAAQ,WAAW,GAAG,aAAa,QAAQ;AACtD;AAeO,SAAS,YAAY,QAA2B;AACrD,SAAO,OAAO,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,GAAG,IAAI,MAAM;AACvD;AAQO,SAAS,qBACd,WACA,UACA,aACA,oBACA,OAAuF,CAAC,GACjE;AACvB,QAAM,MAA6B,CAAC;AACpC,aAAW,OAAO,oBAAoB;AACpC,UAAM,SAAS,YAAY,WAAW,UAAU,aAAa,CAAC,MAAM,EAAE,WAAW,GAAG,CAAC;AACrF,QAAI,OAAO,OAAO,WAAW,EAAG;AAChC,UAAM,YAAY,KAAK,aAAa,OAAO,YAAY,CAAC,GAAG,OAAO,QAAQ,GAAG,OAAO,KAAK,CAAC;AAC1F,UAAM,YAAY,gBAAgB,OAAO,QAAQ,OAAO,OAAO;AAAA,MAC7D,YAAY,KAAK,cAAc;AAAA,MAC/B,WAAW,KAAK,aAAa;AAAA,MAC7B,WAAW;AAAA,MACX,MAAM,KAAK,QAAQ;AAAA,IACrB,CAAC;AACD,QAAI,KAAK;AAAA,MACP,WAAW;AAAA,MACX;AAAA,MACA,WAAW,UAAU,MAAM,CAAC;AAAA,MAC5B;AAAA,MACA,GAAG,OAAO,OAAO;AAAA,IACnB,CAAC;AAAA,EACH;AACA,SAAO;AACT;;;ACjIO,SAAS,sBACd,SAC4B;AAC5B,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,aAAa,QAAQ,cAAc;AACzC,QAAM,YAAY,QAAQ,sBAAsB;AAChD,QAAM,OAAO,QAAQ,iBAAiB;AACtC,QAAM,oBAAoB,QAAQ,qBAAqB;AACvD,QAAM,gBAAgB,QAAQ,8BAA8B;AAE5D,SAAO;AAAA,IACL,MAAM;AAAA,IACN,MAAM,OAAO,KAA6D;AACxE,YAAM,UAAoB,CAAC;AAC3B,YAAM,eAA0E,CAAC;AASjF,YAAM,cAAc,IAAI,IAAI,QAAQ,iBAAiB,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;AACrE,YAAM,MAAM;AAAA,QACV;AAAA,UACE,IAAI;AAAA,UACJ,IAAI,uBAAuB,IAAI;AAAA,UAC/B;AAAA,UACA,CAAC,MAAM,EAAE;AAAA,QACX;AAAA,QACA,EAAE,gBAAgB,mBAAmB,YAAY,WAAW,KAAK;AAAA,MACnE;AACA,YAAM,QAAQ,IAAI,UAAU;AAC5B,YAAM,cAAc,IAAI;AACxB,mBAAa,KAAK;AAAA,QAChB,MAAM;AAAA,QACN,QAAQ;AAAA,QACR,QAAQ;AAAA,UACN,GAAG,IAAI;AAAA,UACP,aAAa,IAAI,UAAU;AAAA,UAC3B,OAAO,IAAI,UAAU;AAAA,UACrB,QAAQ,IAAI,UAAU;AAAA,UACtB,YAAY,IAAI,UAAU;AAAA,UAC1B;AAAA,UACA,SAAS,IAAI;AAAA,QACf;AAAA,MACF,CAAC;AACD,UAAI,CAAC,aAAa;AAChB,gBAAQ;AAAA,UACN,IAAI,UACA,kBAAkB,IAAI,CAAC,mBAAmB,iBAAiB,2CAC3D,mBAAmB,IAAI,UAAU,IAAI,QAAQ,CAAC,CAAC,qBAAgB,cAAc,YAAY,IAAI,UAAU,OAAO,QAAQ,CAAC,CAAC,MAAM,IAAI,UAAU,aAAa,KAAK,QAAQ,CAAC,CAAC,SAAS,IAAI,UAAU,IAAI,QAAQ,CAAC,CAAC,KAAK,IAAI,UAAU,KAAK,QAAQ,CAAC,CAAC;AAAA,QACrP;AAAA,MACF;AAQA,YAAM,UAAU,QAAQ,oBAAoB,SACxC;AAAA,QACE,IAAI;AAAA,QACJ,IAAI,uBAAuB,IAAI;AAAA,QAC/B;AAAA,QACA,QAAQ;AAAA,QACR,EAAE,WAAW,QAAQ,qBAAqB,YAAY,WAAW,KAAK;AAAA,MACxE,IACA,CAAC;AACL,YAAM,YAAY,QAAQ,OAAO,CAAC,MAAM,EAAE,SAAS;AACnD,YAAM,UAAU,UAAU,WAAW;AACrC,mBAAa,KAAK;AAAA,QAChB,MAAM;AAAA,QACN,QAAQ;AAAA,QACR,QAAQ;AAAA,UACN,SAAS,QAAQ,sBAAsB,CAAC;AAAA,UACxC,aAAa,QAAQ,IAAI,CAAC,OAAO;AAAA,YAC/B,WAAW,EAAE;AAAA,YACb,OAAO,EAAE,UAAU;AAAA,YACnB,QAAQ,EAAE,UAAU;AAAA,YACpB,WAAW,EAAE;AAAA,YACb,GAAG,EAAE;AAAA,YACL,WAAW,EAAE;AAAA,UACf,EAAE;AAAA,QACJ;AAAA,MACF,CAAC;AACD,UAAI,CAAC,SAAS;AACZ,gBAAQ;AAAA,UACN,oCAAoC,UAAU,IAAI,CAAC,MAAM,GAAG,EAAE,SAAS,WAAW,EAAE,UAAU,IAAI,QAAQ,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE,KAAK,IAAI,CAAC;AAAA,QAC9I;AAAA,MACF;AAGA,YAAM,aACJ,QAAQ,cAAc,UACtB,IAAI,KAAK,YAAY,IAAI,KAAK,YAAY,QAAQ;AACpD,mBAAa,KAAK;AAAA,QAChB,MAAM;AAAA,QACN,QAAQ;AAAA,QACR,QAAQ;AAAA,UACN,cAAc,IAAI,KAAK;AAAA,UACvB,aAAa,IAAI,KAAK;AAAA,UACtB,WAAW,QAAQ;AAAA,QACrB;AAAA,MACF,CAAC;AACD,UAAI,CAAC,YAAY;AACf,gBAAQ;AAAA,UACN,UAAU,IAAI,KAAK,YAAY,IAAI,KAAK,UAAU,QAAQ,CAAC,CAAC,aAAa,QAAQ,SAAS;AAAA,QAC5F;AAAA,MACF;AAGA,YAAM,kBAAkB,QAAQ,iBAC5B,aAAa,IAAI,oBAAoB,QAAQ,cAAc,IAC3D,EAAE,QAAQ,MAAM,UAAU,CAAC,EAAE;AACjC,mBAAa,KAAK;AAAA,QAChB,MAAM;AAAA,QACN,QAAQ,gBAAgB;AAAA,QACxB,QAAQ;AAAA,UACN,UAAU,gBAAgB,SAAS;AAAA,UACnC,QAAQ,gBAAgB,SAAS,MAAM,GAAG,CAAC;AAAA,QAC7C;AAAA,MACF,CAAC;AACD,UAAI,CAAC,gBAAgB,QAAQ;AAC3B,gBAAQ,KAAK,0BAA0B,gBAAgB,SAAS,MAAM,YAAY;AAAA,MACpF;AAGA,UAAI,sBAAkD;AACtD,UAAI,QAAQ,cAAc,QAAQ,WAAW,UAAU,IAAI;AACzD,8BAAsB,oBAAoB,EAAE,MAAM,QAAQ,WAAW,CAAC;AAAA,MACxE;AAIA,YAAM,kBAAkB;AACxB,YAAM,kBAAkB,qBAAqB,YAAY,CAAC,GAAG;AAAA,QAC3D,CAAC,MAAM,EAAE,YAAY;AAAA,MACvB;AACA,YAAM,oBACJ,CAAC,uBACD,CAAC,iBACA,eAAe,WAAW,KAAK,oBAAoB,YAAY;AAClE,mBAAa,KAAK;AAAA,QAChB,MAAM;AAAA,QACN,QAAQ;AAAA,QACR,QAAQ,EAAE,QAAQ,qBAAqB,oBAAoB,eAAe,OAAO;AAAA,MACnF,CAAC;AACD,UAAI,CAAC,mBAAmB;AACtB,gBAAQ;AAAA,UACN,mCAAmC,eAAe,MAAM,sCAAsC,oBAAqB,OAAO;AAAA,QAC5H;AAAA,MACF;AAGA,UAAI,eAAoC;AACxC,UAAI,QAAQ,cAAc,QAAQ,WAAW,UAAU,IAAI;AACzD,uBAAe,YAAY,QAAQ,YAAY,CAAC,CAAC;AAAA,MACnD;AAEA,YAAM,eAAe,cAAc,UAAU,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,aAAa,OAAO;AACrF,YAAM,aAAa,YAAY,WAAW;AAC1C,mBAAa,KAAK;AAAA,QAChB,MAAM;AAAA,QACN,QAAQ;AAAA,QACR,QAAQ,EAAE,aAAa,cAAc,OAAO,UAAU,GAAG,aAAa,YAAY,OAAO;AAAA,MAC3F,CAAC;AACD,UAAI,CAAC,YAAY;AACf,gBAAQ,KAAK,wBAAwB,YAAY,MAAM,EAAE;AAAA,MAC3D;AAGA,YAAM,YAAY,aAAa,MAAM,CAAC,MAAM,EAAE,MAAM;AACpD,YAAM,WAAW,YAAY,SAAS;AAEtC,aAAO;AAAA,QACL;AAAA,QACA,SAAS,QAAQ,SAAS,IAAI,UAAU,CAAC,kBAAkB;AAAA,QAC3D,mBAAmB;AAAA,QACnB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,aACP,WACA,SAC8E;AAC9E,QAAM,WAA0D,CAAC;AACjE,aAAW,CAAC,SAAS,QAAQ,KAAK,WAAW;AAC3C,UAAM,OAAO,YAAY,QAAQ;AACjC,QAAI,SAAS,OAAW;AACxB,eAAW,UAAU,SAAS;AAC5B,YAAM,UAAU,mBAAmB,MAAM,CAAC,GAAG,MAAM;AACnD,UAAI,CAAC,QAAQ,QAAQ;AACnB,iBAAS,KAAK,EAAE,YAAY,OAAO,IAAI,QAAQ,QAAQ,UAAU,wBAAwB,CAAC;AAAA,MAC5F;AAAA,IACF;AAAA,EACF;AACA,SAAO,EAAE,QAAQ,SAAS,WAAW,GAAG,SAAS;AACnD;AAEA,SAAS,YAAY,UAAuC;AAC1D,MAAI,OAAO,aAAa,SAAU,QAAO;AACzC,MAAI,YAAY,OAAO,aAAa,UAAU;AAC5C,UAAM,MAAM;AACZ,QAAI,OAAO,IAAI,SAAS,SAAU,QAAO,IAAI;AAC7C,QAAI,OAAO,IAAI,WAAW,SAAU,QAAO,IAAI;AAC/C,QAAI,OAAO,IAAI,YAAY,SAAU,QAAO,IAAI;AAAA,EAClD;AACA,SAAO;AACT;;;ACjLO,SAAS,oBACd,KACA,YACA,OAAmC,CAAC,GACpB;AAChB,MAAI,WAAW,WAAW,GAAG;AAC3B,UAAM,IAAI,MAAM,oDAAoD;AAAA,EACtE;AACA,QAAM,oBAAoB,KAAK,qBAAqB;AACpD,QAAM,aAAa,KAAK,cAAc;AACtC,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,OAAO,KAAK,QAAQ;AAC1B,QAAM,WAAW,IAAI,uBAAuB,IAAI;AAChD,QAAM,cAAc,IAAI,IAAI,IAAI,UAAU,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;AAE1D,QAAM,OAAuB,CAAC;AAC9B,aAAW,OAAO,YAAY;AAC5B,QAAI;AACJ,QAAI,IAAI,OAAO,SAAS,aAAa;AACnC,eAAS,CAAC,MAAM,EAAE;AAAA,IACpB,OAAO;AACL,YAAM,MAAM,IAAI,OAAO;AACvB,eAAS,CAAC,MAAM,EAAE,WAAW,GAAG;AAAA,IAClC;AACA,UAAM,SAAS,YAAY,IAAI,aAAa,UAAU,aAAa,MAAM;AAIzE,UAAM,SAAS,IAAI,cAAc,aAAa,OAAO,SAAS,OAAO;AACrE,UAAM,QAAQ,IAAI,cAAc,aAAa,OAAO,QAAQ,OAAO;AACnE,UAAM,YAAY,gBAAgB,QAAQ,OAAO;AAAA,MAC/C;AAAA,MACA;AAAA,MACA,WAAW;AAAA,MACX;AAAA,IACF,CAAC;AACD,UAAM,IAAI,OAAO,OAAO;AACxB,UAAM,iBACJ,IAAI,kBAAkB,OAAO,YAAY,CAAC,GAAG,OAAO,QAAQ,GAAG,OAAO,KAAK,CAAC;AAC9E,UAAM,gBAAgB,IAAI,iBAAiB;AAM3C,UAAM,UACJ,IAAI,oBACA,aACA,UAAU,MAAM,CAAC,iBACf,cACA,UAAU,MAAM,gBACd,aACA;AACV,SAAK,KAAK;AAAA,MACR,MAAM,IAAI;AAAA,MACV,QAAQ,IAAI;AAAA,MACZ,WAAW,IAAI;AAAA,MACf;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AACA,QAAM,KAAK,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC;AACnD,QAAM,OAAO,GAAG,SAAS,IAAI,KAAK,IAAI,GAAG,EAAE,IAAI;AAC/C,SAAO,EAAE,MAAM,MAAM,MAAM,EAAE,WAAW,IAAI,KAAK,WAAW,UAAU,IAAI,KAAK,SAAS,EAAE;AAC5F;AASO,IAAM,eAAgC,CAAC,OAAO;AACnD,QAAM,oBAAoB,GAAG,KAAK,IAAI,CAAC,QAAQ;AAAA,IAC7C,MAAM,aAAa,GAAG,IAAI;AAAA,IAC1B,QAAQ,GAAG,YAAY;AAAA,IACvB,QAAQ;AAAA,MACN,WAAW,GAAG;AAAA,MACd,QAAQ,GAAG;AAAA,MACX,SAAS,GAAG;AAAA,MACZ,GAAG,GAAG;AAAA,MACN,aAAa,GAAG,UAAU;AAAA,MAC1B,OAAO,GAAG,UAAU;AAAA,MACpB,QAAQ,GAAG,UAAU;AAAA,MACrB,YAAY,GAAG,UAAU;AAAA,MACzB,eAAe,GAAG;AAAA,MAClB,gBAAgB,GAAG;AAAA,IACrB;AAAA,EACF,EAAE;AAEF,QAAM,YAAY,GAAG,KAAK,OAAO,CAAC,MAAM,EAAE,YAAY,WAAW;AACjE,QAAM,UAAU,GAAG,KAAK,OAAO,CAAC,MAAM,EAAE,YAAY,UAAU;AAC9D,QAAM,WAAW,GAAG,KAAK,OAAO,CAAC,MAAM,EAAE,YAAY,UAAU;AAE/D,MAAI;AACJ,QAAM,UAAoB,CAAC;AAC3B,MAAI,UAAU,SAAS,GAAG;AAIxB,eAAW;AACX,eAAW,KAAK,WAAW;AACzB,cAAQ;AAAA,QACN,cAAc,EAAE,IAAI,sCAAsC,EAAE,UAAU,IAAI,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,OAAO,EAAE,CAAC;AAAA,MACvH;AAAA,IACF;AAAA,EACF,WAAW,QAAQ,SAAS,GAAG;AAG7B,eAAW;AACX,eAAW,KAAK,SAAS;AACvB,cAAQ;AAAA,QACN,cAAc,EAAE,IAAI,gBAAgB,EAAE,CAAC;AAAA,MACzC;AAAA,IACF;AAAA,EACF,WAAW,SAAS,SAAS,GAAG;AAG9B,eAAW;AACX,YAAQ;AAAA,MACN,+CAA+C,SAC5C;AAAA,QACC,CAAC,MACC,IAAI,EAAE,IAAI,MAAM,EAAE,UAAU,OAAO,QAAQ,CAAC,CAAC,YAAY,EAAE,UAAU,IAAI,QAAQ,CAAC,CAAC;AAAA,MACvF,EACC,KAAK,IAAI,CAAC;AAAA,IACf;AAAA,EACF,OAAO;AAGL,eAAW;AACX,YAAQ;AAAA,MACN;AAAA,IACF;AAAA,EACF;AAIA,QAAM,YAAY,GAAG,KAAK,KAAK,CAAC,MAAM,EAAE,OAAO,SAAS,WAAW,KAAK,GAAG,KAAK,CAAC;AACjF,SAAO,EAAE,UAAU,SAAS,mBAAmB,OAAO,WAAW,UAAU,OAAO;AACpF;AAiBO,SAAS,uBACd,SAC4B;AAC5B,MAAI,QAAQ,WAAW,WAAW,GAAG;AACnC,UAAM,IAAI,MAAM,uDAAuD;AAAA,EACzE;AACA,QAAM,SAAS,QAAQ,UAAU;AACjC,SAAO;AAAA,IACL,MAAM,QAAQ,QAAQ;AAAA,IACtB,MAAM,OAAO,KAA6D;AACxE,YAAM,KAAK,oBAAoB,KAAK,QAAQ,YAAY,OAAO;AAC/D,aAAO,OAAO,EAAE;AAAA,IAClB;AAAA,EACF;AACF;;;ACrQA,eAAsB,QACpB,MAC+C;AAC/C,SAAO,YAAY,IAAI;AACzB;","names":[]}

package/docs/design/external-agent-wedge.md

@@ -1,89 +0,0 @@
-# External-Agent Wedge — plug-in self-improvement for any agent
-
-**Status:** proposed · **Owner:** Drew · **Tracking:** ops-board #507 (epic) · **Updated:** 2026-05-26
-
-> One sentence: **expose the self-improvement engine our own fleet runs on as a drop-in library any agent builder can install — land free, bill the hosted intelligence.**
-
-This doc locks direction so the team executes without thrash. It is deliberately short. If a question isn't answered here, the default is "do the cheapest thing that keeps the LAND tier free and the EXPAND tier billable."
-
----
-
-## Why now
-
-Repeated signal from agent founders (latest: a high-quality GTM/social-marketing agent, sharp CTO): **no evals, no observability, hand-built, no way for the agent to improve itself.** This is the median serious agent team. They don't need another trace viewer — they need their agent to *get better on its own* and proof that it did.
-
-We already built that engine and hardened it the hard way: **6 of our own agents** (tax, legal, creative, gtm, agent-builder, physim) now run the *same* `runCampaign` / `runImprovementLoop` / `gepaDriver` / gate surface. It is `@tangle-network/agent-eval` — already on npm, runtime-agnostic, dogfooded in production. Exposing it externally is distribution of an asset we already own, not a new product to invent.
-
-## Positioning — self-improvement first, not observability
-
-If we sell "observability," we are a worse Langfuse/Braintrust/Arize and it's a margin race. Our wedge is the **closed self-improvement loop** (eval campaign → reflective `gepaDriver` prompt optimization → gated promotion → Bradley-Terry tournaments → predictive-validity calibration). Observability is the *byproduct*, not the pitch.
-
-**The pitch:** "Plug us in. Your agent runs a closed self-improvement loop against your own use case, gated so it never ships a regression — and you get the eval + trace observability for free as it does."
-
-### Backend-agnostic by design — the sandbox is swappable, not required
-
-The whole stack already runs **without our sandbox**, at two granularities:
-- **`agent-eval`** (the self-improvement engine) touches the sandbox only as a *type* — the LAND tier needs no sandbox.
-- **`agent-runtime`** is runtime-decoupled too: every sandbox import is `import type`, and the sandbox backend takes an *injected* instance. Its `Backend` seam already ships `createOpenAICompatibleBackend` (pure LLM) and `createIterableBackend` (bring-your-own) next to `createSandboxPromptBackend`. A builder runs the full runtime + self-improvement loop on **their own execution backend**, no Tangle sandbox installed.
-
-So adoption is *graduated*, and the builder picks the depth: (1) **trace-analysis only** over their existing runtime; (2) the **full self-improvement loop** on their own backend; (3) **our sandbox as a swap-in backend** for the batteries. Our sandbox becomes the best *option*, never the cost of entry. The only work to make this explicit: mark the `@tangle-network/sandbox` peer `optional` in agent-runtime + document the `Backend` interface as the swap seam (Phase A).
-
-## The surface — three tiers (land → expand → platform)
-
-| Tier | What they do | What they get | Billing |
-|---|---|---|---|
-| **LAND** (exists today) | `npm i @tangle-network/agent-eval`, wrap their agent behind one `dispatch` seam, bring a judge | Full self-improvement loop + **local** trace/eval artifacts. Any infra, no sandbox. | Free (lib) — **with optional Tangle Router as a $0-friction inference upsell.** When a builder points `OPENAI_BASE_URL` at `router.tangle.tools/v1`, every campaign call (agent + judge + reflective mutation) routes through us; we earn the routing margin. Same code, opt-in monetization vector that ships today. |
-| **EXPAND** (the build) | Route trace/eval/labeled-scenario data to our orchestrator | Hosted dashboards, cross-run intelligence, the capture flywheel as a service | **Metered** — composes with existing sandbox Stripe + cost-ledger |
-| **PLATFORM** (the carrot) | Move execution into our sandbox (agent-dev-container) | Substrate + orchestrator data/intelligence pre-wired, batteries included | Sandbox usage |
-
-The free lib casts the widest possible net at near-zero cost (it's already published). LAND is **not actually zero-revenue** — pointing the loop at Tangle Router is a one-line config change with no other code differences, so we monetize inference for any LAND-tier adopter who opts in. The wedge ladder is therefore four steps: no-revenue install → router routing margin (LAND with router) → metered data hosting (EXPAND) → sandbox usage (PLATFORM). Each step a one-line config change, never a rewrite. Value capture concentrates at EXPAND (hosting their data/intelligence is the biggest billable surface), but LAND-with-router is the immediate upsell available from day one.
-
-## Plan & gates — land-first, validate, then build
-
-The non-negotiable discipline: **do not build the hosted/billing tier before the free LAND is validated on a foreign agent.** Reality on someone else's real agent is cheaper than our imagination.
-
-- **Phase A — unblock (ungated, now):** upgrade agent-dev-container to the latest substrate; export `OutcomeStore`/`DeploymentOutcome` from `/rl`; mark agent-runtime's `@tangle-network/sandbox` peer `optional` + document the `Backend` swap seam (make sandbox-free adoption explicitly supported). Pure wins, correct regardless of the wedge.
-- **Phase B — design-partner LAND validation (forcing function):** wrap the founder's agent behind `dispatch`, author a marketing-quality judge, run one real campaign + `gepaDriver` loop. Instrument integration friction, judge cold-start, and actual quality lift.
-- **GATE — go/no-go + pricing:** decided from Phase B evidence, not theory.
-- **Phase C — LAND ergonomics:** external 15-minute quickstart + a stable `dispatch`/judge/scenario contract + ≥1 reference framework adapter.
-- **Phase D — EXPAND (gated):** hosted OTLP/eval-run HTTP sink (client in agent-eval) + multi-tenant orchestrator ingest + metered billing + minimal dashboard (server in `@tangle-network/monorepo`).
-
-## Success metrics
-
-- **Phase B:** ≥1 measurable quality lift on the partner's own use case; integration ≤ a 1–2 day pairing.
-- **LAND:** time-to-first-self-improvement-loop for a new external agent < 1 day from the quickstart.
-- **EXPAND:** first external tenant routed + first metered dollar.
-
-## Risk register
-
-**Knowns (high confidence)**
-- The substrate is a portable lib; it wraps our 6 agents behind `dispatch`/judge/gate in production.
-- It's runtime-agnostic (FS + in-memory storage, Node + edge) and emits OTLP traces.
-- **agent-runtime is already sandbox-decoupled at runtime** — sandbox is type-only + dependency-injected; the `Backend` seam (`createOpenAICompatibleBackend` / `createIterableBackend` / `createSandboxPromptBackend`) makes the sandbox one swappable option. Only the peer-dep label needs to change.
-- The orchestrator/observability/billing platform lives in `@tangle-network/monorepo`.
-
-**Known-unknowns (Phase B answers these)**
-- Does `dispatch` wrap a *foreign* agent as cleanly as ours, or are there hidden Tangle assumptions?
-- Judge cold-start: can a team with no prior evals author a usable judge for a subjective domain (marketing quality)?
-- Data cold-start: no evals = no scenarios = no labels; how do they bootstrap the flywheel day 1?
-- Orchestrator multi-tenancy: it's only run internally — external auth/isolation/privacy is unbuilt.
-- Pricing line + OSS boundary (the lib is already public).
-
-**Unknown-unknowns (instrument, don't predict)**
-- Integration-surface explosion → keep a tiny stable contract + reference adapters; refuse to special-case.
-- Foreign-domain eval semantics we've never seen → the design partner is the discovery mechanism.
-- Multi-tenant orchestrator failure modes at external/adversarial scale.
-- Supply-chain/trust — we enter their dependency tree; our security posture becomes their procurement question.
-- *Mitigation for all:* land-first-with-a-partner surfaces surprises cheaply, on a real agent, before any hosted spend. Timebox Phase D behind the gate.
-
-## Non-goals (anti-thrash)
-
-- **Not** a standalone observability product. Observability ships only as a byproduct of self-improvement.
-- **Not** building the hosted/billing tier before Phase B validates the lib.
-- **Not** per-framework bespoke integrations — one stable contract + a couple of reference adapters.
-- **Not** re-architecting the substrate for hypothetical external needs — extend at the `dispatch`/judge/store seams only.
-- **Not** changing the sandbox/products roadmap — this *exposes the same engine*, it doesn't fork it.
-
-## Tracking
-
-ops-board epic **#507** + children: agent-dev-container stack upgrade (eng), foreign-agent adoption surface (eng), design-partner LAND validation (gtm), hosted orchestrator routing + billing (eng, blocked on the gate), GTM-fit + pricing decision (gtm).

package/docs/design/phase-d-rfc.md

@@ -1,125 +0,0 @@
-# Phase D RFC — hosted-tier substrate
-
-Pinned scope decisions for the EXPAND tier. What we built, what we
-deliberately did NOT, and what's gated on Phase B evidence.
-
----
-
-## What's in this version
-
-**Wire-format substrate (shipped):**
-
-1. `@tangle-network/agent-eval/hosted` — public client + types for shipping
-   eval-run events + trace spans to any orchestrator that speaks the wire
-   format.
-2. `docs/hosted-ingest-spec.md` — semver-committed wire spec
-   (`HostedWireVersion = "2026-05-26.v1"`).
-3. `examples/hosted-ingest-server/` — minimal hono-based reference
-   receiver (~200 LOC). Executable spec. Stays as the reference even
-   after the production orchestrator ships.
-4. `selfImprove({ hostedTenant })` opt-in — when set, the substrate
-   POSTs the final eval-run event to the configured endpoint. Failures
-   are logged but never fail the loop (LAND tier never blocks on
-   EXPAND-tier infra).
-
-**Production orchestrator (started):**
-
-5. HTTP ingest service in `@tangle-network/monorepo` accepting the wire
-   format. Lives under the orchestrator app. Tenant auth + isolation
-   + persistent storage + read endpoints. *Started this session — see
-   the @tangle-network/agent-dev-container PR. Not feature-complete:
-   tenant CRUD + adversarial isolation tests pending.*
-
-## What's deliberately deferred
-
-The wedge doc gates these on Phase B evidence — partner-validated
-signal about what the hosted product actually needs to do. Shipping
-them without that signal risks building the wrong thing.
-
-| Deferred until Phase B passes | Why |
-|---|---|
-| **Metered billing wire-up (Stripe + cost-ledger)** | The billable units (per-eval-run, per-ingested-MB, per-seat) depend on actual partner consumption patterns. Picking dimensions in a vacuum locks us into wrong pricing. |
-| **Multi-tenant dashboard UX** | Partners' first dashboard request defines the right default views. We have a stub list-runs page; the rest is post-signal. |
-| **Webhook callbacks per tenant** | The events partners want pushed (gate-decided, cost-threshold, regression-alert) are partner-shaped. Add them when a partner asks. |
-| **Cross-tenant aggregation / benchmarking** | This is the "Datadog for agents" tier — explicit roadmap, requires user volume we don't have. |
-| **Sandbox-cost roll-up into hosted billing** | Cross-product billing integration requires PLATFORM-tier partners. Out of scope until at least one. |
-| **Trace UI** | OTel-shape spans store fine. Visualization comes after partners ask. Phoenix / Jaeger / any OTLP-compatible viewer covers it in the interim. |
-| **Soc2 / compliance audit work** | Required for enterprise; not required for design partners. |
-
-## Architecture decisions locked
-
-These are committed and won't change without a major-version wire bump
-or a documented migration:
-
-1. **Wire format is JSON over HTTP**, not gRPC. Reasons: works in
-   browsers + edge + node + curl; OTel-compatible at the trace stream
-   level; lowest possible barrier to a self-hosted orchestrator.
-2. **Tenant auth is bearer-token + tenant-id header**, not OIDC /
-   service-account / mutual-TLS. Reasons: simplest thing that's
-   actually secure with proper key handling; defers complex IAM until
-   enterprise demand.
-3. **Idempotency via header, not transactional API**. Servers MUST
-   dedupe by `(tenantId, Idempotency-Key)` for 24h. Simpler than
-   making clients commit transactions.
-4. **Eval-runs and traces are SEPARATE streams** with pivot keys
-   (`tangle.runId` etc.) on spans. Reasons: traces can be best-effort
-   (lossy) without corrupting eval-run semantics; orchestrators can
-   prioritize eval-run durability without forcing trace durability.
-5. **Wire version is a date.v-N string**, not semver. Reasons: dates
-   communicate "when was this contract frozen"; v-N captures
-   incremental breaking changes between dates.
-
-## Open questions for Phase B to answer
-
-When the design-partner pairing happens, capture answers to these
-explicitly:
-
-1. **Surface confidentiality**: do partners want the verbatim surface
-   (system prompt) shipped, or just the hash? Today the wire format
-   has `surface?` as optional; partner default is what we ship.
-2. **Trace sampling**: at what cells-per-second do trace spans become
-   noise? What's the right default sampling rate?
-3. **Cost attribution granularity**: per cell? per generation? per
-   run? Per judge dimension? Partner needs determine what we surface
-   in billing reports.
-4. **Replay**: do partners want to re-run an old eval-run from the
-   stored data? That would require us to store more than the summary —
-   actual artifacts + prompts. Storage cost implication.
-5. **PII / sensitive scenarios**: how do partners want to handle
-   scenarios containing user data? Encryption-at-rest is table stakes;
-   redaction-at-ingest may be required for some.
-
-The partner pairing kit (`docs/phase-b-pairing-kit.md`) has discovery
-questions that probe these.
-
-## Non-goals (explicit)
-
-This RFC does NOT plan for:
-
-- Replacing Langfuse / Phoenix / Arize. We INGEST OTel; we don't
-  build a generic trace viewer. The dashboard is eval-run-shaped, not
-  trace-shaped.
-- Becoming a model gateway. Tangle Router exists; the hosted
-  orchestrator routes to Tangle Router by default but doesn't
-  duplicate its function.
-- Becoming an LLM-call CDN. Caching is the consumer's job (their
-  agent code, their HTTP client). We don't intercept LLM calls.
-- Building an "agents IDE." Substrate, not surface.
-
-## Migration path (post Phase B)
-
-When Phase B passes the gate, the production orchestrator finishes:
-
-1. Replace in-memory store with Postgres (tenant data) + S3 (large
-   artifacts) OR Cloudflare D1 + R2 (Workers-native).
-2. Wire metered events to Stripe + the cost-ledger.
-3. Tenant CRUD UI + onboarding flow.
-4. Multi-tenant dashboard MVP (list runs, drill into one, diff
-   generations, view shipped prompt).
-5. Adversarial tenant-isolation test battery in CI.
-6. Webhooks + observability for the orchestrator itself.
-
-Estimated effort post-Phase-B: ~1 week focused work for one engineer.
-This is fast precisely BECAUSE the wire format is locked and the
-reference receiver exists — the production server is a different
-implementation of the same contract.

package/docs/design/phase4-consumer-migration.md

@@ -1,70 +0,0 @@
-# Phase 4 — consumer migration tracking
-
-Migrate the product repos off their duplicated eval / prompt-evolution
-orchestration onto the published substrate (`@tangle-network/agent-eval@^0.40.3`
-+ `@tangle-network/agent-runtime@^0.25.0`). Integration contract:
-[`primitives-integration-spec.md`](./primitives-integration-spec.md).
-
-**Strategy:** prove **gtm end-to-end first** (the canonical consumer), then fan
-the proven migration pattern to the rest via parallel subagents, each briefed
-with the gtm reference diff + the spec's forbidden-anti-patterns list. Each
-migration is its own reviewable, rollback-able PR.
-
-## Status board
-
-| Repo | Deletable orchestration (LOC est.) | Dispatch seam | Status | PR |
-|---|---|---|---|---|
-| gtm-agent | ~2,420 | `runChatThroughRuntime` | **IN PROGRESS** | — |
-| legal-agent | tbd | tbd | queued | — |
-| tax-agent | tbd | tbd | queued | — |
-| creative-agent | tbd | tbd | queued | — |
-| agent-builder | tbd | tbd | queued | — |
-| blueprint-agent | tbd | tbd | queued (Drew dispatching via spec) | — |
-| physim | tbd (MultiLayerVerifier adapter) | tbd | queued | — |
-
-## Per-repo migration checklist
-
-For each repo, in order:
-
-- [ ] **Survey** — inventory eval + prompt-evolution wrappers (file:line + LOC).
-      Identify the dispatch seam, scenarios, judges, mutation strategy.
-- [ ] **Bump deps** — `@tangle-network/agent-eval` → `^0.40.0`,
-      `@tangle-network/agent-runtime` → `^0.25.0`; `pnpm update`; baseline
-      typecheck green.
-- [ ] **Rewire seams** — `dispatch`/`dispatchWithSurface`, `judges`,
-      `scenarios` extracted from the existing wrappers (KEEP domain logic).
-- [ ] **Replace orchestration** — swap the local generation/population/scorecard
-      loop for `runImprovementLoop` (or `runCampaign` for eval-only). DELETE the
-      wrapper body.
-- [ ] **Gate** — compose domain gates with `defaultProductionGate`.
-- [ ] **Dataset** — wire `FsLabeledScenarioStore` with correct `captureSource`.
-- [ ] **Tests** — port wrapper contract tests to assert the substrate wiring;
-      keep judge/scenario tests. Suite green.
-- [ ] **Prove** — one real eval/improve run end-to-end; confirm scorecard +
-      (if applicable) a PR opens on a shipping gate.
-- [ ] **Anti-pattern sweep** — no silent fallbacks, no reimplemented loop, no
-      train/holdout conflation, tracing on, dispatch named.
-- [ ] **PR** — open, independent-review, merge.
-
-## gtm-agent — migration map (from survey)
-
-- **Branch base:** off the repo's working branch (`feat/gtm-rich-chat-actions`)
-  or main — confirm before starting.
-- **Dispatch seam:** `runChatThroughRuntime(ctx)`
-  (`src/lib/.server/agent-runtime/chat.ts`) — prompt variant + scenario → real
-  agent run → artifact + events + token usage.
-- **Scenarios:** `src/lib/.server/production-loop/scenarios.ts` (3 holdout) +
-  `eval/business-owner/personas.json` (canonical personas).
-- **Judges:** `src/lib/.server/production-loop/judges.ts` (`runEnsembleJudge`,
-  3-model ensemble) + canonical 12-dimension judges in `eval/canonical.ts`.
-- **Delete (~2,420 LOC orchestration):** the generation/population/reps loop in
-  `src/lib/.server/production-loop/index.ts` (~450), the checkpoint loop in
-  `eval/canonical.ts` (~600), `eval/run-prompt-evolution.ts` wrapper (~800),
-  `eval/analyst-loop.ts` wrapper (~300), `eval/optimization-campaign.ts` (~170),
-  `scripts/evals/run-optimization-campaign.ts` (~100 scaffold).
-- **Rewire:** `buildHoldoutRunner` → `dispatchWithSurface`; `buildScorer` →
-  `judges`; `buildMutator` → `evolutionaryDriver({ mutator })`;
-  `runProductionLoop` → `runImprovementLoop`.
-- **Keep:** judges, scenarios, persona data + reactive driver, deterministic
-  anti-slop/brief checks, GitHub PR wiring, feedback/trace ingestion.
-- **Net:** ~1,400–1,600 LOC reduction.