@tangle-network/agent-eval 0.122.9 → 0.123.0

package/dist/benchmarks/index.js

@@ -17,8 +17,8 @@ import {
   runBenchmarkAdapter,
   summarizeBenchmarkCampaign
 } from "../chunk-FHFTYX2Q.js";
-import "../chunk-CWF6SCBK.js";
-import "../chunk-4EDDIK77.js";
+import "../chunk-GS3FJGUF.js";
+import "../chunk-HZJF4IUO.js";
 import "../chunk-3FCG7FBV.js";
 import "../chunk-FC5NDO3E.js";
 import "../chunk-ARU2PZFM.js";

package/dist/campaign/index.d.ts

@@ -4338,6 +4338,15 @@ type RunImprovementLoopOptions<TScenario extends Scenario, TArtifact> = RunOptim
     /** Holdout scenarios kept OUT of the training optimization pool — used
      *  ONLY to score baseline vs winner for the gate. */
     holdoutScenarios: TScenario[];
+    /** Holdout policy. Default `'measured'`: baseline + winner are re-scored on
+     *  `holdoutScenarios` and the gate decides on that held-out comparison.
+     *  `'deferred'`: the improvement-set (search) campaigns run exactly as usual,
+     *  but ZERO holdout cells are dispatched, the gate is forced to `'hold'`, and
+     *  the result + provenance record carry `holdout: 'deferred'` with NO
+     *  held-out lift — for callers that measure the held-out comparison in a
+     *  separate later run instead of faking a static holdout scenario and
+     *  recording a meaningless lift. */
+    holdout?: 'measured' | 'deferred';
     /** Promotion gate. Substrate strongly recommends `defaultProductionGate`
      *  for production wiring (composes red-team / reward-hacking / canary /
      *  heldout). */
@@ -4365,6 +4374,10 @@ interface RunImprovementLoopResult<TArtifact, TScenario extends Scenario> extend
     neutralizedOnHoldout?: CampaignResult<TArtifact, TScenario>;
     neutralizedSurface?: MutableSurface;
     gateResult: Awaited<ReturnType<Gate<TArtifact, TScenario>['decide']>>;
+    /** Present iff the loop ran with `holdout: 'deferred'`. When set,
+     *  `baselineOnHoldout`/`winnerOnHoldout` are the shared EMPTY campaign (zero
+     *  cells dispatched) and the gate verdict is the forced `'hold'`. */
+    holdout?: 'deferred';
     /** Unified baseline→winner surface diff. Computed UNCONDITIONALLY (not only
      *  when `autoOnPromote === 'pr'`) so the diff that the gate decided on is
      *  always present on the result + in the emitted provenance record. Empty
@@ -6938,12 +6951,18 @@ interface LoopProvenanceRecord {
             detail: unknown;
         }>;
     };
-    /** baseline-on-holdout composite mean. */
-    baselineHoldoutComposite: number;
-    /** winner-on-holdout composite mean. */
-    winnerHoldoutComposite: number;
-    /** winnerHoldout - baselineHoldout — RECOMPUTABLE from this record. */
-    heldOutLift: number;
+    /** Present iff the loop ran with `holdout: 'deferred'` — the held-out
+     *  comparison was intentionally not measured in this run, so the holdout
+     *  composites and `heldOutLift` are ABSENT rather than recorded as a
+     *  meaningless 0. */
+    holdout?: 'deferred';
+    /** baseline-on-holdout composite mean. Absent when `holdout === 'deferred'`. */
+    baselineHoldoutComposite?: number;
+    /** winner-on-holdout composite mean. Absent when `holdout === 'deferred'`. */
+    winnerHoldoutComposite?: number;
+    /** winnerHoldout - baselineHoldout — RECOMPUTABLE from this record. Absent
+     *  when `holdout === 'deferred'` (no held-out measurement ran). */
+    heldOutLift?: number;
     /** Backend provenance: stub-vs-real verdict + worker call count + models. */
     backend: LoopProvenanceBackend;
     totalCostUsd: number;
@@ -6973,6 +6992,10 @@ interface BuildLoopProvenanceArgs<TArtifact, TScenario extends Scenario> {
         }>;
     }>;
     gate: GateResult;
+    /** Holdout policy the loop ran with. `'deferred'` ⇒ the holdout campaigns
+     *  below are the shared empty campaign and the record omits the holdout
+     *  composites + `heldOutLift`. Default `'measured'`. */
+    holdout?: 'measured' | 'deferred';
     baselineOnHoldout: CampaignResult<TArtifact, TScenario>;
     winnerOnHoldout: CampaignResult<TArtifact, TScenario>;
     neutralizedSurface?: MutableSurface;

package/dist/campaign/index.js

@@ -68,7 +68,7 @@ import {
   userStoryScoreboard,
   validateSearchLedgerEvent,
   verifyCodeSurface
-} from "../chunk-CWF6SCBK.js";
+} from "../chunk-GS3FJGUF.js";
 import {
   assertCodeSurfaceIdentity,
   buildEvidenceVector,
@@ -109,7 +109,7 @@ import {
   surfaceContentHash,
   surfaceHash,
   verifyLoopProvenanceRecord
-} from "../chunk-4EDDIK77.js";
+} from "../chunk-HZJF4IUO.js";
 import {
   SearchLedgerConflictError,
   SearchLedgerError,

package/dist/{chunk-CWF6SCBK.js → chunk-GS3FJGUF.js}

@@ -15,7 +15,7 @@ import {
   runImprovementLoop,
   surfaceContentHash,
   surfaceHash
-} from "./chunk-4EDDIK77.js";
+} from "./chunk-HZJF4IUO.js";
 import {
   SearchLedgerConflictError,
   SearchLedgerError,
@@ -8279,4 +8279,4 @@ export {
   verifyCodeSurface,
   resolveWorktreePath
 };
-//# sourceMappingURL=chunk-CWF6SCBK.js.map
+//# sourceMappingURL=chunk-GS3FJGUF.js.map

package/dist/{chunk-4EDDIK77.js → chunk-HZJF4IUO.js}

@@ -3006,7 +3006,18 @@ async function runImprovementLoop(opts) {
   const optimization = await runOptimization({ ...opts, dispatchTimeoutMs, costLedger });
   const winnerIsBaseline = optimization.winnerSurfaceHash === surfaceHash(opts.baselineSurface);
   const { runCampaign: runCampaign2 } = await import("./run-campaign-ZKR5MQMQ.js");
-  const baselineOnHoldout = await runCampaign2({
+  const holdoutDeferred = (opts.holdout ?? "measured") === "deferred";
+  const baselineOnHoldout = holdoutDeferred ? await runCampaign2({
+    ...opts,
+    costLedger,
+    costPhase: "holdout.deferred",
+    dispatchTimeoutMs,
+    scenarios: [],
+    dispatch: async () => {
+      throw new Error("runImprovementLoop: unreachable dispatch \u2014 holdout is deferred");
+    },
+    runDir: `${opts.runDir}/holdout-deferred`
+  }) : await runCampaign2({
     ...opts,
     costLedger,
     costPhase: "holdout.baseline",
@@ -3015,7 +3026,7 @@ async function runImprovementLoop(opts) {
     dispatch: (scenario, ctx) => opts.dispatchWithSurface(opts.baselineSurface, scenario, ctx),
     runDir: `${opts.runDir}/holdout-baseline`
   });
-  const winnerOnHoldout = winnerIsBaseline ? baselineOnHoldout : await runCampaign2({
+  const winnerOnHoldout = winnerIsBaseline || holdoutDeferred ? baselineOnHoldout : await runCampaign2({
     ...opts,
     costLedger,
     costPhase: "holdout.winner",
@@ -3039,8 +3050,10 @@ async function runImprovementLoop(opts) {
       );
     }
   };
-  assertCompleteHoldout("baseline", baselineOnHoldout);
-  assertCompleteHoldout("winner", winnerOnHoldout);
+  if (!holdoutDeferred) {
+    assertCompleteHoldout("baseline", baselineOnHoldout);
+    assertCompleteHoldout("winner", winnerOnHoldout);
+  }
   const candidateArtifacts = /* @__PURE__ */ new Map();
   const baselineArtifacts = /* @__PURE__ */ new Map();
   const judgeScores = /* @__PURE__ */ new Map();
@@ -3057,7 +3070,7 @@ async function runImprovementLoop(opts) {
   let neutralizedJudgeScores;
   let neutralizedOnHoldout;
   let neutralizedSurface;
-  if (opts.neutralize && !winnerIsBaseline) {
+  if (opts.neutralize && !winnerIsBaseline && !holdoutDeferred) {
     const surface = opts.neutralize(optimization.winnerSurface, opts.baselineSurface);
     neutralizedSurface = surface;
     neutralizedOnHoldout = await runCampaign2({
@@ -3077,7 +3090,15 @@ async function runImprovementLoop(opts) {
       neutralizedJudgeScores.set(cell.cellId, cell.judgeScores);
     }
   }
-  const gateResult = winnerIsBaseline ? {
+  const gateResult = holdoutDeferred ? {
+    decision: "hold",
+    reasons: [
+      "holdout deferred \u2014 improvement-set search completed without a held-out measurement; nothing to promote from this run"
+    ],
+    contributingGates: [
+      { name: "holdout-deferred", passed: false, detail: { holdout: "deferred" } }
+    ]
+  } : winnerIsBaseline ? {
     decision: "hold",
     reasons: [
       "no candidate beat the training baseline \u2014 winner == baseline (empty diff); nothing to promote"
@@ -3118,6 +3139,7 @@ async function runImprovementLoop(opts) {
     baselineOnHoldout,
     winnerOnHoldout,
     ...neutralizedOnHoldout && neutralizedSurface ? { neutralizedOnHoldout, neutralizedSurface } : {},
+    ...holdoutDeferred ? { holdout: "deferred" } : {},
     gateResult,
     promotedDiff,
     prResult,
@@ -3170,6 +3192,7 @@ function loopProvenanceArgsFromResult(input) {
       }))
     })),
     gate: result.gateResult,
+    ...result.holdout === "deferred" ? { holdout: "deferred" } : {},
     baselineOnHoldout: result.baselineOnHoldout,
     winnerOnHoldout: result.winnerOnHoldout,
     ...result.neutralizedSurface && result.neutralizedOnHoldout ? {
@@ -3305,6 +3328,7 @@ function buildLoopProvenanceRecord(args) {
       "buildLoopProvenanceRecord: winner surface does not match the final promoted incumbent"
     );
   }
+  const holdoutDeferred = args.holdout === "deferred";
   const baselineHoldoutComposite = meanHoldoutComposite(args.baselineOnHoldout);
   const winnerHoldoutComposite = meanHoldoutComposite(args.winnerOnHoldout);
   if (args.baselineOnHoldout.splitDigest !== args.winnerOnHoldout.splitDigest) {
@@ -3364,9 +3388,13 @@ function buildLoopProvenanceRecord(args) {
         detail: durableGateDetail(g.detail)
       }))
     },
-    baselineHoldoutComposite,
-    winnerHoldoutComposite,
-    heldOutLift: winnerHoldoutComposite - baselineHoldoutComposite,
+    // Deferred holdout records the MODE, never a fabricated 0-lift: the
+    // held-out comparison intentionally did not run in this loop.
+    ...holdoutDeferred ? { holdout: "deferred" } : {
+      baselineHoldoutComposite,
+      winnerHoldoutComposite,
+      heldOutLift: winnerHoldoutComposite - baselineHoldoutComposite
+    },
     backend: {
       verdict: integrity.verdict,
       workerCallCount: integrity.totalRecords,
@@ -3537,24 +3565,26 @@ function loopProvenanceSpans(record, opts = {}) {
   const endNano = baseNano + Math.max(1, record.totalDurationMs) * 1e6;
   const spans = [];
   const rootSpanId = hashId(["root", record.runId]).slice(0, 16);
+  const rootAttributes = {
+    "tangle.runId": record.runId,
+    "tangle.runDir": record.runDir,
+    "tangle.baselineContentHash": record.baselineContentHash,
+    "tangle.winnerContentHash": record.winnerContentHash,
+    "tangle.baselineSearchComposite": record.baselineSearchComposite,
+    "tangle.gateDecision": record.gate.decision,
+    "tangle.backendVerdict": record.backend.verdict,
+    "tangle.workerCallCount": record.backend.workerCallCount,
+    "tangle.totalCostUsd": record.totalCostUsd
+  };
+  if (record.heldOutLift !== void 0) rootAttributes["tangle.heldOutLift"] = record.heldOutLift;
+  if (record.holdout) rootAttributes["tangle.holdout"] = record.holdout;
   spans.push({
     traceId,
     spanId: rootSpanId,
     name: "improvement-loop",
     startTimeUnixNano: baseNano,
     endTimeUnixNano: endNano,
-    attributes: {
-      "tangle.runId": record.runId,
-      "tangle.runDir": record.runDir,
-      "tangle.baselineContentHash": record.baselineContentHash,
-      "tangle.winnerContentHash": record.winnerContentHash,
-      "tangle.baselineSearchComposite": record.baselineSearchComposite,
-      "tangle.heldOutLift": record.heldOutLift,
-      "tangle.gateDecision": record.gate.decision,
-      "tangle.backendVerdict": record.backend.verdict,
-      "tangle.workerCallCount": record.backend.workerCallCount,
-      "tangle.totalCostUsd": record.totalCostUsd
-    },
+    attributes: rootAttributes,
     status: gateStatus(record.gate.decision),
     "tangle.runId": record.runId
   });
@@ -3625,6 +3655,21 @@ function loopProvenanceSpans(record, opts = {}) {
     }
   }
   const gateSpanId = hashId(["gate", record.runId]).slice(0, 16);
+  const gateAttributes = {
+    "tangle.runId": record.runId,
+    "tangle.gateDecision": record.gate.decision,
+    "tangle.gateReasons": JSON.stringify(record.gate.reasons)
+  };
+  const gateDelta = record.gate.delta ?? record.heldOutLift;
+  if (gateDelta !== void 0) gateAttributes["tangle.gateDelta"] = gateDelta;
+  if (record.heldOutLift !== void 0) gateAttributes["tangle.heldOutLift"] = record.heldOutLift;
+  if (record.baselineHoldoutComposite !== void 0) {
+    gateAttributes["tangle.baselineHoldoutComposite"] = record.baselineHoldoutComposite;
+  }
+  if (record.winnerHoldoutComposite !== void 0) {
+    gateAttributes["tangle.winnerHoldoutComposite"] = record.winnerHoldoutComposite;
+  }
+  if (record.holdout) gateAttributes["tangle.holdout"] = record.holdout;
   spans.push({
     traceId,
     spanId: gateSpanId,
@@ -3632,15 +3677,7 @@ function loopProvenanceSpans(record, opts = {}) {
     name: "gate-decision",
     startTimeUnixNano: endNano,
     endTimeUnixNano: endNano,
-    attributes: {
-      "tangle.runId": record.runId,
-      "tangle.gateDecision": record.gate.decision,
-      "tangle.gateDelta": record.gate.delta ?? record.heldOutLift,
-      "tangle.gateReasons": JSON.stringify(record.gate.reasons),
-      "tangle.heldOutLift": record.heldOutLift,
-      "tangle.baselineHoldoutComposite": record.baselineHoldoutComposite,
-      "tangle.winnerHoldoutComposite": record.winnerHoldoutComposite
-    },
+    attributes: gateAttributes,
     status: gateStatus(record.gate.decision),
     "tangle.runId": record.runId
   });
@@ -3695,7 +3732,9 @@ function buildEvalRunEvent(args, record) {
       snapshotFromHoldout(1, record.winnerContentHash, args.winnerSurface, args.winnerOnHoldout)
     ],
     gateDecision: args.gate.decision,
-    holdoutLift: record.heldOutLift,
+    // Deferred holdout ships NO lift — the wire field is optional and a 0
+    // would read downstream as a measured no-improvement.
+    ...record.heldOutLift !== void 0 ? { holdoutLift: record.heldOutLift } : {},
     totalCostUsd: args.totalCostUsd,
     totalDurationMs: args.totalDurationMs
   };
@@ -3797,4 +3836,4 @@ export {
   provenanceSpansPath,
   emitLoopProvenance
 };
-//# sourceMappingURL=chunk-4EDDIK77.js.map
+//# sourceMappingURL=chunk-HZJF4IUO.js.map