@tangle-network/agent-eval 0.65.0 → 0.67.0

package/dist/chunk-MZ2IYGGN.js

@@ -0,0 +1,592 @@
+import {
+  runCanaries,
+  scoreRedTeamOutput
+} from "./chunk-NV2PF37Q.js";
+import {
+  runCampaign,
+  summarizeBackendIntegrity
+} from "./chunk-6XQIEUQ2.js";
+import {
+  detectRewardHacking
+} from "./chunk-YV7J7X5N.js";
+import {
+  pairedBootstrap
+} from "./chunk-ITBRCT73.js";
+
+// src/campaign/drivers/evolutionary.ts
+function evolutionaryDriver(opts) {
+  return {
+    kind: `evolutionary:${opts.mutator.kind}`,
+    async propose({ currentSurface, findings, populationSize, signal }) {
+      return opts.mutator.mutate({
+        findings: findings.length > 0 ? findings : opts.findings ?? [],
+        currentSurface,
+        populationSize,
+        signal
+      });
+    }
+  };
+}
+
+// src/campaign/gates/compose.ts
+function composeGate(...gates) {
+  if (gates.length === 0) {
+    throw new Error("composeGate requires at least one gate");
+  }
+  return {
+    name: `composed(${gates.map((g) => g.name).join(",")})`,
+    async decide(ctx) {
+      const results = [];
+      for (const gate of gates) {
+        const res = await gate.decide(ctx);
+        results.push({ gate, res });
+      }
+      const decisions = results.map((r) => r.res.decision);
+      const overall = decisions.every((d) => d === "ship") ? "ship" : decisions.includes("arch_ceiling") ? "arch_ceiling" : decisions.includes("model_ceiling") ? "model_ceiling" : decisions.includes("hold") ? "hold" : "need_more_work";
+      const contributing = results.flatMap(
+        (r) => r.res.contributingGates.length > 0 ? r.res.contributingGates : [{ name: r.gate.name, passed: r.res.decision === "ship", detail: r.res }]
+      );
+      const reasons = results.flatMap(
+        (r) => r.res.reasons.map((reason) => `[${r.gate.name}] ${reason}`)
+      );
+      return {
+        decision: overall,
+        reasons,
+        contributingGates: contributing,
+        delta: results[0]?.res.delta
+      };
+    }
+  };
+}
+
+// src/campaign/gates/statistical-heldout.ts
+function pairHoldout(candidate, baseline, scenarioIds, select) {
+  const cellValue = (byCell, cellId) => {
+    const scores = byCell.get(cellId);
+    if (!scores) return void 0;
+    const vals = [];
+    for (const s of Object.values(scores)) {
+      const v = select(s);
+      if (typeof v === "number" && Number.isFinite(v)) vals.push(v);
+    }
+    if (vals.length === 0) return void 0;
+    return vals.reduce((a, b) => a + b, 0) / vals.length;
+  };
+  const inScope = (cellId) => scenarioIds.has(cellId.split(":")[0] ?? "");
+  const candCells = [...candidate.keys()].filter(inScope).sort();
+  const baseCells = [...baseline.keys()].filter(inScope).sort();
+  if (candCells.length !== baseCells.length || candCells.some((c, i) => c !== baseCells[i])) {
+    throw new Error(
+      `pairHoldout: candidate/baseline holdout cells do not align \u2014 candidate=[${candCells.join(",")}] baseline=[${baseCells.join(",")}]. Both holdout campaigns must run the same scenarios with the same seed base.`
+    );
+  }
+  const before = [];
+  const after = [];
+  const cellIds = [];
+  for (const cellId of candCells) {
+    const b = cellValue(baseline, cellId);
+    const a = cellValue(candidate, cellId);
+    if (b === void 0 || a === void 0) continue;
+    before.push(b);
+    after.push(a);
+    cellIds.push(cellId);
+  }
+  return { before, after, cellIds };
+}
+function heldoutSignificance(paired, opts = {}) {
+  const deltaThreshold = opts.deltaThreshold ?? 0;
+  const minProductiveRuns = opts.minProductiveRuns ?? 3;
+  const bootstrap = pairedBootstrap(paired.before, paired.after, {
+    confidence: opts.confidence ?? 0.95,
+    resamples: opts.resamples ?? 2e3,
+    statistic: opts.statistic ?? "median",
+    seed: opts.seed ?? 1337
+  });
+  const n = paired.before.length;
+  const fewRuns = n < minProductiveRuns;
+  const significant = !fewRuns && bootstrap.low > deltaThreshold;
+  return { paired, bootstrap, n, significant, fewRuns };
+}
+function detectScale(values) {
+  return values.some((v) => Math.abs(v) > 1.5) ? 100 : 1;
+}
+function dimensionRegressions(candidate, baseline, scenarioIds, criticalDimensions, opts = {}) {
+  const out = [];
+  for (const dim of criticalDimensions) {
+    const paired = pairHoldout(candidate, baseline, scenarioIds, (s) => s.dimensions[dim]);
+    if (paired.before.length === 0) continue;
+    const tolerance = opts.tolerance ?? 0.05 * detectScale([...paired.before, ...paired.after]);
+    const bootstrap = pairedBootstrap(paired.before, paired.after, {
+      confidence: opts.confidence ?? 0.95,
+      resamples: opts.resamples ?? 2e3,
+      statistic: "median",
+      seed: opts.seed ?? 1337
+    });
+    out.push({
+      dimension: dim,
+      bootstrap,
+      regressed: bootstrap.low < -tolerance,
+      tolerance,
+      n: paired.before.length
+    });
+  }
+  return out;
+}
+
+// src/campaign/gates/default-production-gate.ts
+function defaultProductionGate(options) {
+  const deltaThreshold = options.deltaThreshold ?? 0;
+  const confidence = options.confidence ?? 0.95;
+  const resamples = options.bootstrapResamples ?? 2e3;
+  const seed = options.bootstrapSeed ?? 1337;
+  const minProductiveRuns = options.minProductiveRuns ?? 3;
+  const blockOnGaming = options.blockOnRewardHackingGaming ?? true;
+  return {
+    name: "defaultProductionGate",
+    async decide(ctx) {
+      const reasons = [];
+      const contributing = [];
+      const scenarioIds = new Set(options.holdoutScenarios.map((s) => s.id));
+      const sig = heldoutSignificance(
+        pairHoldout(
+          ctx.judgeScores,
+          ctx.baselineJudgeScores ?? ctx.judgeScores,
+          scenarioIds,
+          (s) => s.composite
+        ),
+        { deltaThreshold, minProductiveRuns, confidence, resamples, seed }
+      );
+      const delta = sig.bootstrap.median;
+      const heldoutPass = sig.significant;
+      contributing.push({
+        name: "heldout-significance",
+        passed: heldoutPass,
+        detail: {
+          n: sig.n,
+          deltaMedian: sig.bootstrap.median,
+          ciLow: sig.bootstrap.low,
+          ciHigh: sig.bootstrap.high,
+          confidence: sig.bootstrap.confidence,
+          deltaThreshold,
+          fewRuns: sig.fewRuns
+        }
+      });
+      if (!heldoutPass) {
+        reasons.push(
+          sig.fewRuns ? `held-out: only ${sig.n} paired runs (< ${minProductiveRuns}) \u2014 too few to claim significance` : `held-out CI.low ${sig.bootstrap.low.toFixed(3)} \u2264 threshold ${deltaThreshold} (median ${sig.bootstrap.median.toFixed(3)}, ${(sig.bootstrap.confidence * 100).toFixed(0)}% CI [${sig.bootstrap.low.toFixed(3)}, ${sig.bootstrap.high.toFixed(3)}])`
+        );
+      }
+      const dimRegs = options.criticalDimensions?.length ? dimensionRegressions(
+        ctx.judgeScores,
+        ctx.baselineJudgeScores ?? ctx.judgeScores,
+        scenarioIds,
+        options.criticalDimensions,
+        { tolerance: options.regressionTolerance, confidence, resamples, seed }
+      ) : [];
+      const regressed = dimRegs.filter((d) => d.regressed);
+      const dimPass = regressed.length === 0;
+      contributing.push({
+        name: "dimension-regression",
+        passed: dimPass,
+        detail: {
+          guarded: options.criticalDimensions ?? [],
+          regressions: dimRegs.map((d) => ({
+            dimension: d.dimension,
+            ciLow: d.bootstrap.low,
+            median: d.bootstrap.median,
+            tolerance: d.tolerance,
+            n: d.n,
+            regressed: d.regressed
+          }))
+        }
+      });
+      if (!dimPass) {
+        reasons.push(
+          `critical dimension(s) regressed: ${regressed.map((d) => `${d.dimension} CI.low ${d.bootstrap.low.toFixed(3)} < -${d.tolerance}`).join("; ")}`
+        );
+      }
+      const budgetPass = options.budgetUsd === void 0 || ctx.cost.candidate + ctx.cost.baseline <= options.budgetUsd;
+      contributing.push({
+        name: "budget",
+        passed: budgetPass,
+        detail: {
+          candidateUsd: ctx.cost.candidate,
+          baselineUsd: ctx.cost.baseline,
+          budgetUsd: options.budgetUsd
+        }
+      });
+      if (!budgetPass) {
+        reasons.push(
+          `spend ${(ctx.cost.candidate + ctx.cost.baseline).toFixed(2)} > budget ${options.budgetUsd}`
+        );
+      }
+      const redTeamFindings = options.redTeamBattery ? probeRedTeam(ctx.candidateArtifacts, options.redTeamBattery) : { passed: true, findings: [] };
+      contributing.push({
+        name: "red-team",
+        passed: redTeamFindings.passed,
+        detail: {
+          failures: redTeamFindings.findings.length,
+          sample: redTeamFindings.findings.slice(0, 3)
+        }
+      });
+      if (!redTeamFindings.passed) {
+        reasons.push(`red-team probe failed (${redTeamFindings.findings.length} findings)`);
+      }
+      let rewardHackingReport = null;
+      if (options.recentRuns && options.recentRuns.length >= 10) {
+        rewardHackingReport = detectRewardHacking({ runs: options.recentRuns });
+      }
+      const gamingThreshold = 0.6;
+      const gamingFindings = (rewardHackingReport?.findings ?? []).filter(
+        (f) => f.severity >= gamingThreshold
+      );
+      const rewardHackingPass = !rewardHackingReport || !blockOnGaming || gamingFindings.length === 0 && rewardHackingReport.verdict !== "gaming";
+      contributing.push({
+        name: "reward-hacking",
+        passed: rewardHackingPass,
+        detail: { report: rewardHackingReport, gamingFindingCount: gamingFindings.length }
+      });
+      if (!rewardHackingPass) {
+        reasons.push(
+          `reward-hacking detector flagged ${gamingFindings.length} gaming-severity findings (verdict=${rewardHackingReport.verdict})`
+        );
+      }
+      let canaryReport = null;
+      if (options.recentRuns && options.recentRuns.length >= 10) {
+        canaryReport = runCanaries(options.recentRuns, {});
+      }
+      const errorAlerts = (canaryReport?.alerts ?? []).filter((a) => a.severity === "error");
+      const canaryPass = errorAlerts.length === 0;
+      contributing.push({
+        name: "canary",
+        passed: canaryPass,
+        detail: { totalAlerts: canaryReport?.alerts.length ?? 0, errorAlerts: errorAlerts.length }
+      });
+      if (!canaryPass) {
+        reasons.push(`canary error alerts: ${errorAlerts.length}`);
+      }
+      const allPassed = contributing.every((c) => c.passed);
+      const decision = allPassed ? "ship" : "hold";
+      return {
+        decision,
+        reasons: reasons.length > 0 ? reasons : ["all gates passed"],
+        contributingGates: contributing,
+        delta
+      };
+    }
+  };
+}
+function probeRedTeam(artifacts, battery) {
+  const findings = [];
+  for (const [_cellId, artifact] of artifacts) {
+    const text = extractText(artifact);
+    if (text === void 0) continue;
+    for (const rtCase of battery) {
+      const finding = scoreRedTeamOutput(text, [], rtCase);
+      if (!finding.passed) {
+        findings.push({ scenarioId: rtCase.id, reason: finding.reason ?? "red-team probe failed" });
+      }
+    }
+  }
+  return { passed: findings.length === 0, findings };
+}
+function extractText(artifact) {
+  if (typeof artifact === "string") return artifact;
+  if (artifact && typeof artifact === "object") {
+    const rec = artifact;
+    if (typeof rec.text === "string") return rec.text;
+    if (typeof rec.output === "string") return rec.output;
+    if (typeof rec.content === "string") return rec.content;
+  }
+  return void 0;
+}
+
+// src/campaign/presets/run-eval.ts
+async function runEval(opts) {
+  return runCampaign(opts);
+}
+
+// src/campaign/provenance.ts
+import { createHash } from "crypto";
+import { join } from "path";
+function surfaceContentHash(surface) {
+  const material = typeof surface === "string" ? surface : JSON.stringify({
+    kind: surface.kind,
+    worktreeRef: surface.worktreeRef,
+    baseRef: surface.baseRef ?? null
+  });
+  return `sha256:${createHash("sha256").update(material).digest("hex")}`;
+}
+function meanHoldoutComposite(campaign) {
+  const xs = [];
+  for (const cell of campaign.cells) {
+    if (cell.error) continue;
+    const cs = Object.values(cell.judgeScores).map((s) => s.composite);
+    if (cs.length) xs.push(cs.reduce((a, b) => a + b, 0) / cs.length);
+  }
+  return xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : 0;
+}
+function buildLoopProvenanceRecord(args) {
+  const integrity = summarizeBackendIntegrity(args.workerRecords);
+  const models = [...new Set(args.workerRecords.map((r) => r.model))].sort();
+  const candidates = [];
+  for (const gen of args.generations) {
+    const promotedSet = new Set(gen.promoted);
+    const surfaceByHash = new Map(gen.surfaces.map((s) => [s.surfaceHash, s.surface]));
+    for (const c of gen.candidates) {
+      const surface = surfaceByHash.get(c.surfaceHash);
+      const entry = {
+        generation: gen.generationIndex,
+        surfaceHash: c.surfaceHash,
+        contentHash: surface !== void 0 ? surfaceContentHash(surface) : `sha256:${c.surfaceHash}`,
+        composite: c.composite,
+        promoted: promotedSet.has(c.surfaceHash)
+      };
+      if (c.label) entry.label = c.label;
+      if (c.rationale) entry.rationale = c.rationale;
+      candidates.push(entry);
+    }
+  }
+  const baselineHoldoutComposite = meanHoldoutComposite(args.baselineOnHoldout);
+  const winnerHoldoutComposite = meanHoldoutComposite(args.winnerOnHoldout);
+  const record = {
+    schema: "tangle.loop-provenance.v1",
+    runId: args.runId,
+    runDir: args.runDir,
+    timestamp: args.timestamp,
+    baselineContentHash: surfaceContentHash(args.baselineSurface),
+    winnerContentHash: surfaceContentHash(args.winnerSurface),
+    diff: args.diff,
+    candidates,
+    gate: {
+      decision: args.gate.decision,
+      reasons: args.gate.reasons,
+      delta: args.gate.delta,
+      contributingGates: args.gate.contributingGates.map((g) => ({
+        name: g.name,
+        passed: g.passed
+      }))
+    },
+    baselineHoldoutComposite,
+    winnerHoldoutComposite,
+    heldOutLift: winnerHoldoutComposite - baselineHoldoutComposite,
+    backend: {
+      verdict: integrity.verdict,
+      workerCallCount: integrity.totalRecords,
+      models,
+      totalInputTokens: integrity.totalInputTokens,
+      totalOutputTokens: integrity.totalOutputTokens,
+      totalCostUsd: integrity.totalCostUsd
+    },
+    totalCostUsd: args.totalCostUsd,
+    totalDurationMs: args.totalDurationMs
+  };
+  if (args.winnerLabel) record.winnerLabel = args.winnerLabel;
+  if (args.winnerRationale) record.winnerRationale = args.winnerRationale;
+  return record;
+}
+var DECISION_OK = ["ship"];
+function hashId(parts) {
+  return createHash("sha256").update(parts.join(":")).digest("hex");
+}
+function gateStatus(decision) {
+  return DECISION_OK.includes(decision) ? { code: "OK" } : { code: "ERROR", message: `gate decision: ${decision}` };
+}
+function loopProvenanceSpans(record, opts = {}) {
+  const traceId = hashId(["trace", record.runId]).slice(0, 32);
+  const baseNano = (opts.baseTimeMs ?? (Date.parse(record.timestamp) || Date.now())) * 1e6;
+  const endNano = baseNano + Math.max(1, record.totalDurationMs) * 1e6;
+  const spans = [];
+  const rootSpanId = hashId(["root", record.runId]).slice(0, 16);
+  spans.push({
+    traceId,
+    spanId: rootSpanId,
+    name: "improvement-loop",
+    startTimeUnixNano: baseNano,
+    endTimeUnixNano: endNano,
+    attributes: {
+      "tangle.runId": record.runId,
+      "tangle.runDir": record.runDir,
+      "tangle.baselineContentHash": record.baselineContentHash,
+      "tangle.winnerContentHash": record.winnerContentHash,
+      "tangle.heldOutLift": record.heldOutLift,
+      "tangle.gateDecision": record.gate.decision,
+      "tangle.backendVerdict": record.backend.verdict,
+      "tangle.workerCallCount": record.backend.workerCallCount,
+      "tangle.totalCostUsd": record.totalCostUsd
+    },
+    status: gateStatus(record.gate.decision),
+    "tangle.runId": record.runId
+  });
+  const byGen = /* @__PURE__ */ new Map();
+  for (const c of record.candidates) {
+    const arr = byGen.get(c.generation) ?? [];
+    arr.push(c);
+    byGen.set(c.generation, arr);
+  }
+  for (const [generation, cands] of [...byGen.entries()].sort((a, b) => a[0] - b[0])) {
+    const genSpanId = hashId(["gen", record.runId, String(generation)]).slice(0, 16);
+    const bestComposite = cands.reduce((m, c) => Math.max(m, c.composite), 0);
+    spans.push({
+      traceId,
+      spanId: genSpanId,
+      parentSpanId: rootSpanId,
+      name: `generation-${generation}`,
+      startTimeUnixNano: baseNano,
+      endTimeUnixNano: endNano,
+      attributes: {
+        "tangle.runId": record.runId,
+        "tangle.generation": generation,
+        "tangle.populationSize": cands.length,
+        "tangle.bestComposite": bestComposite
+      },
+      "tangle.runId": record.runId,
+      "tangle.generation": generation
+    });
+    for (let i = 0; i < cands.length; i++) {
+      const c = cands[i];
+      const candSpanId = hashId(["cand", record.runId, String(generation), c.surfaceHash]).slice(
+        0,
+        16
+      );
+      const attributes = {
+        "tangle.runId": record.runId,
+        "tangle.generation": generation,
+        "tangle.surfaceHash": c.surfaceHash,
+        "tangle.contentHash": c.contentHash,
+        "tangle.composite": c.composite,
+        "tangle.promoted": c.promoted
+      };
+      if (c.label) attributes["tangle.candidateLabel"] = c.label;
+      if (c.rationale) attributes["tangle.candidateRationale"] = c.rationale;
+      spans.push({
+        traceId,
+        spanId: candSpanId,
+        parentSpanId: genSpanId,
+        name: `candidate-${c.surfaceHash}`,
+        startTimeUnixNano: baseNano,
+        endTimeUnixNano: endNano,
+        attributes,
+        "tangle.runId": record.runId,
+        "tangle.generation": generation
+      });
+    }
+  }
+  const gateSpanId = hashId(["gate", record.runId]).slice(0, 16);
+  spans.push({
+    traceId,
+    spanId: gateSpanId,
+    parentSpanId: rootSpanId,
+    name: "gate-decision",
+    startTimeUnixNano: endNano,
+    endTimeUnixNano: endNano,
+    attributes: {
+      "tangle.runId": record.runId,
+      "tangle.gateDecision": record.gate.decision,
+      "tangle.gateDelta": record.gate.delta ?? record.heldOutLift,
+      "tangle.gateReasons": JSON.stringify(record.gate.reasons),
+      "tangle.heldOutLift": record.heldOutLift,
+      "tangle.baselineHoldoutComposite": record.baselineHoldoutComposite,
+      "tangle.winnerHoldoutComposite": record.winnerHoldoutComposite
+    },
+    status: gateStatus(record.gate.decision),
+    "tangle.runId": record.runId
+  });
+  return spans;
+}
+function provenanceRecordPath(runDir) {
+  return join(runDir, "loop-provenance.json");
+}
+function provenanceSpansPath(runDir) {
+  return join(runDir, "loop-provenance-spans.jsonl");
+}
+function snapshotFromHoldout(index, surfaceHash, surface, campaign) {
+  const cells = campaign.cells.map((cell) => {
+    const judgeScores = Object.values(cell.judgeScores);
+    const composite = judgeScores.length === 0 ? 0 : judgeScores.reduce((s, j) => s + j.composite, 0) / judgeScores.length;
+    const score = {
+      scenarioId: cell.scenarioId,
+      rep: cell.rep,
+      compositeMean: composite,
+      dimensions: Object.fromEntries(
+        Object.entries(cell.judgeScores).map(([name, s]) => [name, s.dimensions])
+      )
+    };
+    if (cell.error) score.errorMessage = cell.error;
+    return score;
+  });
+  const compositeMean = cells.length === 0 ? 0 : cells.reduce((s, c) => s + c.compositeMean, 0) / cells.length;
+  return {
+    index,
+    surfaceHash,
+    surface,
+    cells,
+    compositeMean,
+    costUsd: campaign.aggregates.totalCostUsd,
+    durationMs: campaign.durationMs
+  };
+}
+function buildEvalRunEvent(args, record) {
+  return {
+    runId: args.runId,
+    runDir: args.runDir,
+    timestamp: args.timestamp,
+    status: "finished",
+    labels: {},
+    baseline: snapshotFromHoldout(
+      0,
+      record.baselineContentHash,
+      args.baselineSurface,
+      args.baselineOnHoldout
+    ),
+    generations: [
+      snapshotFromHoldout(1, record.winnerContentHash, args.winnerSurface, args.winnerOnHoldout)
+    ],
+    gateDecision: args.gate.decision,
+    holdoutLift: record.heldOutLift,
+    totalCostUsd: args.totalCostUsd,
+    totalDurationMs: args.totalDurationMs
+  };
+}
+async function emitLoopProvenance(args) {
+  const record = buildLoopProvenanceRecord(args);
+  const spans = loopProvenanceSpans(record);
+  args.storage.ensureDir(args.runDir);
+  const recordPath = provenanceRecordPath(args.runDir);
+  const spansPath = provenanceSpansPath(args.runDir);
+  args.storage.write(recordPath, JSON.stringify(record, null, 2));
+  args.storage.write(spansPath, spans.map((s) => JSON.stringify(s)).join("\n"));
+  if (args.hostedClient) {
+    try {
+      await args.hostedClient.ingestEvalRun(buildEvalRunEvent(args, record));
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.warn(`[agent-eval] hosted eval-run ingest failed (continuing): ${msg}`);
+    }
+    try {
+      await args.hostedClient.ingestTraces(spans);
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.warn(`[agent-eval] provenance span ingest failed (continuing): ${msg}`);
+    }
+  }
+  return { record, spans, recordPath, spansPath };
+}
+
+export {
+  evolutionaryDriver,
+  composeGate,
+  pairHoldout,
+  heldoutSignificance,
+  detectScale,
+  dimensionRegressions,
+  defaultProductionGate,
+  runEval,
+  surfaceContentHash,
+  buildLoopProvenanceRecord,
+  loopProvenanceSpans,
+  provenanceRecordPath,
+  provenanceSpansPath,
+  emitLoopProvenance
+};
+//# sourceMappingURL=chunk-MZ2IYGGN.js.map