@tangle-network/agent-eval 0.66.0 → 0.68.0

package/dist/{chunk-Q56RRLEC.js → chunk-JFGZPUMU.js}

@@ -1,5 +1,6 @@
 import {
-  runCampaign
+  runCampaign,
+  summarizeBackendIntegrity
 } from "./chunk-6XQIEUQ2.js";
 import {
   DEFAULT_REDACTION_RULES
@@ -1362,6 +1363,7 @@ async function runImprovementLoop(opts) {
   }
   const dispatchTimeoutMs = opts.dispatchTimeoutMs ?? DEFAULT_DISPATCH_TIMEOUT_MS;
   const optimization = await runOptimization({ ...opts, dispatchTimeoutMs });
+  const winnerIsBaseline = optimization.winnerSurfaceHash === surfaceHash(opts.baselineSurface);
   const { runCampaign: runCampaign2 } = await import("./run-campaign-BVY3RGAZ.js");
   const baselineOnHoldout = await runCampaign2({
     ...opts,
@@ -1370,7 +1372,7 @@ async function runImprovementLoop(opts) {
     dispatch: (scenario, ctx) => opts.dispatchWithSurface(opts.baselineSurface, scenario, ctx),
     runDir: `${opts.runDir}/holdout-baseline`
   });
-  const winnerOnHoldout = await runCampaign2({
+  const winnerOnHoldout = winnerIsBaseline ? baselineOnHoldout : await runCampaign2({
     ...opts,
     dispatchTimeoutMs,
     scenarios: opts.holdoutScenarios,
@@ -1398,7 +1400,16 @@ async function runImprovementLoop(opts) {
     baselineArtifacts.set(cell.cellId, cell.artifact);
     baselineJudgeScores.set(cell.cellId, cell.judgeScores);
   }
-  const gateResult = await opts.gate.decide({
+  const gateResult = winnerIsBaseline ? {
+    decision: "hold",
+    reasons: [
+      "no candidate beat the training baseline \u2014 winner == baseline (empty diff); nothing to promote"
+    ],
+    contributingGates: [
+      { name: "no-op-guard", passed: false, detail: { winnerIsBaseline: true } }
+    ],
+    delta: 0
+  } : await opts.gate.decide({
     candidateArtifacts,
     baselineArtifacts,
     judgeScores,
@@ -1448,6 +1459,273 @@ ${fmt(winnerSurface)}`;
   return lines.join("\n");
 }
 
+// src/campaign/provenance.ts
+import { createHash as createHash2 } from "crypto";
+import { join as join2 } from "path";
+function surfaceContentHash(surface) {
+  const material = typeof surface === "string" ? surface : JSON.stringify({
+    kind: surface.kind,
+    worktreeRef: surface.worktreeRef,
+    baseRef: surface.baseRef ?? null
+  });
+  return `sha256:${createHash2("sha256").update(material).digest("hex")}`;
+}
+function meanHoldoutComposite(campaign) {
+  const xs = [];
+  for (const cell of campaign.cells) {
+    if (cell.error) continue;
+    const cs = Object.values(cell.judgeScores).map((s) => s.composite);
+    if (cs.length) xs.push(cs.reduce((a, b) => a + b, 0) / cs.length);
+  }
+  return xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : 0;
+}
+function buildLoopProvenanceRecord(args) {
+  const integrity = summarizeBackendIntegrity(args.workerRecords);
+  const models = [...new Set(args.workerRecords.map((r) => r.model))].sort();
+  const candidates = [];
+  for (const gen of args.generations) {
+    const promotedSet = new Set(gen.promoted);
+    const surfaceByHash = new Map(gen.surfaces.map((s) => [s.surfaceHash, s.surface]));
+    for (const c of gen.candidates) {
+      const surface = surfaceByHash.get(c.surfaceHash);
+      const entry = {
+        generation: gen.generationIndex,
+        surfaceHash: c.surfaceHash,
+        contentHash: surface !== void 0 ? surfaceContentHash(surface) : `sha256:${c.surfaceHash}`,
+        composite: c.composite,
+        promoted: promotedSet.has(c.surfaceHash)
+      };
+      if (c.label) entry.label = c.label;
+      if (c.rationale) entry.rationale = c.rationale;
+      candidates.push(entry);
+    }
+  }
+  const baselineHoldoutComposite = meanHoldoutComposite(args.baselineOnHoldout);
+  const winnerHoldoutComposite = meanHoldoutComposite(args.winnerOnHoldout);
+  const record = {
+    schema: "tangle.loop-provenance.v1",
+    runId: args.runId,
+    runDir: args.runDir,
+    timestamp: args.timestamp,
+    baselineContentHash: surfaceContentHash(args.baselineSurface),
+    winnerContentHash: surfaceContentHash(args.winnerSurface),
+    diff: args.diff,
+    candidates,
+    gate: {
+      decision: args.gate.decision,
+      reasons: args.gate.reasons,
+      delta: args.gate.delta,
+      contributingGates: args.gate.contributingGates.map((g) => ({
+        name: g.name,
+        passed: g.passed
+      }))
+    },
+    baselineHoldoutComposite,
+    winnerHoldoutComposite,
+    heldOutLift: winnerHoldoutComposite - baselineHoldoutComposite,
+    backend: {
+      verdict: integrity.verdict,
+      workerCallCount: integrity.totalRecords,
+      models,
+      totalInputTokens: integrity.totalInputTokens,
+      totalOutputTokens: integrity.totalOutputTokens,
+      totalCostUsd: integrity.totalCostUsd
+    },
+    totalCostUsd: args.totalCostUsd,
+    totalDurationMs: args.totalDurationMs
+  };
+  if (args.winnerLabel) record.winnerLabel = args.winnerLabel;
+  if (args.winnerRationale) record.winnerRationale = args.winnerRationale;
+  return record;
+}
+var DECISION_OK = ["ship"];
+function hashId(parts) {
+  return createHash2("sha256").update(parts.join(":")).digest("hex");
+}
+function gateStatus(decision) {
+  return DECISION_OK.includes(decision) ? { code: "OK" } : { code: "ERROR", message: `gate decision: ${decision}` };
+}
+function loopProvenanceSpans(record, opts = {}) {
+  const traceId = hashId(["trace", record.runId]).slice(0, 32);
+  const baseNano = (opts.baseTimeMs ?? (Date.parse(record.timestamp) || Date.now())) * 1e6;
+  const endNano = baseNano + Math.max(1, record.totalDurationMs) * 1e6;
+  const spans = [];
+  const rootSpanId = hashId(["root", record.runId]).slice(0, 16);
+  spans.push({
+    traceId,
+    spanId: rootSpanId,
+    name: "improvement-loop",
+    startTimeUnixNano: baseNano,
+    endTimeUnixNano: endNano,
+    attributes: {
+      "tangle.runId": record.runId,
+      "tangle.runDir": record.runDir,
+      "tangle.baselineContentHash": record.baselineContentHash,
+      "tangle.winnerContentHash": record.winnerContentHash,
+      "tangle.heldOutLift": record.heldOutLift,
+      "tangle.gateDecision": record.gate.decision,
+      "tangle.backendVerdict": record.backend.verdict,
+      "tangle.workerCallCount": record.backend.workerCallCount,
+      "tangle.totalCostUsd": record.totalCostUsd
+    },
+    status: gateStatus(record.gate.decision),
+    "tangle.runId": record.runId
+  });
+  const byGen = /* @__PURE__ */ new Map();
+  for (const c of record.candidates) {
+    const arr = byGen.get(c.generation) ?? [];
+    arr.push(c);
+    byGen.set(c.generation, arr);
+  }
+  for (const [generation, cands] of [...byGen.entries()].sort((a, b) => a[0] - b[0])) {
+    const genSpanId = hashId(["gen", record.runId, String(generation)]).slice(0, 16);
+    const bestComposite = cands.reduce((m, c) => Math.max(m, c.composite), 0);
+    spans.push({
+      traceId,
+      spanId: genSpanId,
+      parentSpanId: rootSpanId,
+      name: `generation-${generation}`,
+      startTimeUnixNano: baseNano,
+      endTimeUnixNano: endNano,
+      attributes: {
+        "tangle.runId": record.runId,
+        "tangle.generation": generation,
+        "tangle.populationSize": cands.length,
+        "tangle.bestComposite": bestComposite
+      },
+      "tangle.runId": record.runId,
+      "tangle.generation": generation
+    });
+    for (let i = 0; i < cands.length; i++) {
+      const c = cands[i];
+      const candSpanId = hashId(["cand", record.runId, String(generation), c.surfaceHash]).slice(
+        0,
+        16
+      );
+      const attributes = {
+        "tangle.runId": record.runId,
+        "tangle.generation": generation,
+        "tangle.surfaceHash": c.surfaceHash,
+        "tangle.contentHash": c.contentHash,
+        "tangle.composite": c.composite,
+        "tangle.promoted": c.promoted
+      };
+      if (c.label) attributes["tangle.candidateLabel"] = c.label;
+      if (c.rationale) attributes["tangle.candidateRationale"] = c.rationale;
+      spans.push({
+        traceId,
+        spanId: candSpanId,
+        parentSpanId: genSpanId,
+        name: `candidate-${c.surfaceHash}`,
+        startTimeUnixNano: baseNano,
+        endTimeUnixNano: endNano,
+        attributes,
+        "tangle.runId": record.runId,
+        "tangle.generation": generation
+      });
+    }
+  }
+  const gateSpanId = hashId(["gate", record.runId]).slice(0, 16);
+  spans.push({
+    traceId,
+    spanId: gateSpanId,
+    parentSpanId: rootSpanId,
+    name: "gate-decision",
+    startTimeUnixNano: endNano,
+    endTimeUnixNano: endNano,
+    attributes: {
+      "tangle.runId": record.runId,
+      "tangle.gateDecision": record.gate.decision,
+      "tangle.gateDelta": record.gate.delta ?? record.heldOutLift,
+      "tangle.gateReasons": JSON.stringify(record.gate.reasons),
+      "tangle.heldOutLift": record.heldOutLift,
+      "tangle.baselineHoldoutComposite": record.baselineHoldoutComposite,
+      "tangle.winnerHoldoutComposite": record.winnerHoldoutComposite
+    },
+    status: gateStatus(record.gate.decision),
+    "tangle.runId": record.runId
+  });
+  return spans;
+}
+function provenanceRecordPath(runDir) {
+  return join2(runDir, "loop-provenance.json");
+}
+function provenanceSpansPath(runDir) {
+  return join2(runDir, "loop-provenance-spans.jsonl");
+}
+function snapshotFromHoldout(index, surfaceHash2, surface, campaign) {
+  const cells = campaign.cells.map((cell) => {
+    const judgeScores = Object.values(cell.judgeScores);
+    const composite = judgeScores.length === 0 ? 0 : judgeScores.reduce((s, j) => s + j.composite, 0) / judgeScores.length;
+    const score = {
+      scenarioId: cell.scenarioId,
+      rep: cell.rep,
+      compositeMean: composite,
+      dimensions: Object.fromEntries(
+        Object.entries(cell.judgeScores).map(([name, s]) => [name, s.dimensions])
+      )
+    };
+    if (cell.error) score.errorMessage = cell.error;
+    return score;
+  });
+  const compositeMean = cells.length === 0 ? 0 : cells.reduce((s, c) => s + c.compositeMean, 0) / cells.length;
+  return {
+    index,
+    surfaceHash: surfaceHash2,
+    surface,
+    cells,
+    compositeMean,
+    costUsd: campaign.aggregates.totalCostUsd,
+    durationMs: campaign.durationMs
+  };
+}
+function buildEvalRunEvent(args, record) {
+  return {
+    runId: args.runId,
+    runDir: args.runDir,
+    timestamp: args.timestamp,
+    status: "finished",
+    labels: {},
+    baseline: snapshotFromHoldout(
+      0,
+      record.baselineContentHash,
+      args.baselineSurface,
+      args.baselineOnHoldout
+    ),
+    generations: [
+      snapshotFromHoldout(1, record.winnerContentHash, args.winnerSurface, args.winnerOnHoldout)
+    ],
+    gateDecision: args.gate.decision,
+    holdoutLift: record.heldOutLift,
+    totalCostUsd: args.totalCostUsd,
+    totalDurationMs: args.totalDurationMs
+  };
+}
+async function emitLoopProvenance(args) {
+  const record = buildLoopProvenanceRecord(args);
+  const spans = loopProvenanceSpans(record);
+  args.storage.ensureDir(args.runDir);
+  const recordPath = provenanceRecordPath(args.runDir);
+  const spansPath = provenanceSpansPath(args.runDir);
+  args.storage.write(recordPath, JSON.stringify(record, null, 2));
+  args.storage.write(spansPath, spans.map((s) => JSON.stringify(s)).join("\n"));
+  if (args.hostedClient) {
+    try {
+      await args.hostedClient.ingestEvalRun(buildEvalRunEvent(args, record));
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.warn(`[agent-eval] hosted eval-run ingest failed (continuing): ${msg}`);
+    }
+    try {
+      await args.hostedClient.ingestTraces(spans);
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.warn(`[agent-eval] provenance span ingest failed (continuing): ${msg}`);
+    }
+  }
+  return { record, spans, recordPath, spansPath };
+}
+
 export {
   dominates,
   paretoFrontier,
@@ -1478,6 +1756,12 @@ export {
   runOptimization,
   surfaceHash,
   runImprovementLoop,
-  defaultRenderDiff
+  defaultRenderDiff,
+  surfaceContentHash,
+  buildLoopProvenanceRecord,
+  loopProvenanceSpans,
+  provenanceRecordPath,
+  provenanceSpansPath,
+  emitLoopProvenance
 };
-//# sourceMappingURL=chunk-Q56RRLEC.js.map
+//# sourceMappingURL=chunk-JFGZPUMU.js.map