@tangle-network/agent-eval 0.122.8 → 0.123.0

package/dist/contract/index.d.ts

@@ -2386,6 +2386,15 @@ type RunImprovementLoopOptions<TScenario extends Scenario, TArtifact> = RunOptim
     /** Holdout scenarios kept OUT of the training optimization pool — used
      *  ONLY to score baseline vs winner for the gate. */
     holdoutScenarios: TScenario[];
+    /** Holdout policy. Default `'measured'`: baseline + winner are re-scored on
+     *  `holdoutScenarios` and the gate decides on that held-out comparison.
+     *  `'deferred'`: the improvement-set (search) campaigns run exactly as usual,
+     *  but ZERO holdout cells are dispatched, the gate is forced to `'hold'`, and
+     *  the result + provenance record carry `holdout: 'deferred'` with NO
+     *  held-out lift — for callers that measure the held-out comparison in a
+     *  separate later run instead of faking a static holdout scenario and
+     *  recording a meaningless lift. */
+    holdout?: 'measured' | 'deferred';
     /** Promotion gate. Substrate strongly recommends `defaultProductionGate`
      *  for production wiring (composes red-team / reward-hacking / canary /
      *  heldout). */
@@ -2413,6 +2422,10 @@ interface RunImprovementLoopResult<TArtifact, TScenario extends Scenario> extend
     neutralizedOnHoldout?: CampaignResult<TArtifact, TScenario>;
     neutralizedSurface?: MutableSurface;
     gateResult: Awaited<ReturnType<Gate<TArtifact, TScenario>['decide']>>;
+    /** Present iff the loop ran with `holdout: 'deferred'`. When set,
+     *  `baselineOnHoldout`/`winnerOnHoldout` are the shared EMPTY campaign (zero
+     *  cells dispatched) and the gate verdict is the forced `'hold'`. */
+    holdout?: 'deferred';
     /** Unified baseline→winner surface diff. Computed UNCONDITIONALLY (not only
      *  when `autoOnPromote === 'pr'`) so the diff that the gate decided on is
      *  always present on the result + in the emitted provenance record. Empty
@@ -3715,12 +3728,18 @@ interface LoopProvenanceRecord {
             detail: unknown;
         }>;
     };
-    /** baseline-on-holdout composite mean. */
-    baselineHoldoutComposite: number;
-    /** winner-on-holdout composite mean. */
-    winnerHoldoutComposite: number;
-    /** winnerHoldout - baselineHoldout — RECOMPUTABLE from this record. */
-    heldOutLift: number;
+    /** Present iff the loop ran with `holdout: 'deferred'` — the held-out
+     *  comparison was intentionally not measured in this run, so the holdout
+     *  composites and `heldOutLift` are ABSENT rather than recorded as a
+     *  meaningless 0. */
+    holdout?: 'deferred';
+    /** baseline-on-holdout composite mean. Absent when `holdout === 'deferred'`. */
+    baselineHoldoutComposite?: number;
+    /** winner-on-holdout composite mean. Absent when `holdout === 'deferred'`. */
+    winnerHoldoutComposite?: number;
+    /** winnerHoldout - baselineHoldout — RECOMPUTABLE from this record. Absent
+     *  when `holdout === 'deferred'` (no held-out measurement ran). */
+    heldOutLift?: number;
     /** Backend provenance: stub-vs-real verdict + worker call count + models. */
     backend: LoopProvenanceBackend;
     totalCostUsd: number;
@@ -3767,8 +3786,22 @@ interface SelfImproveBudget {
     holdoutFraction?: number;
     /** Explicit held-out scenarios; overrides `holdoutFraction`. */
     holdoutScenarios?: Scenario[];
+    /** Holdout policy. Default `'measured'`: split, re-score baseline vs winner
+     *  on the held-out set, gate on that comparison. `'deferred'`: run the
+     *  improvement-set campaigns + search promotion, dispatch ZERO holdout cells,
+     *  force the gate to `'hold'`, return `lift: undefined`, and record
+     *  `holdout: 'deferred'` in the provenance record — for callers that measure
+     *  the held-out comparison in a separate later run instead of faking a
+     *  static holdout scenario and recording a meaningless lift. Unless
+     *  `holdoutScenarios` reserves an explicit set, ALL scenarios train. */
+    holdout?: 'measured' | 'deferred';
     /** Per-scenario replicates per cell — raises bootstrap-CI tightness. Default 1. */
     reps?: number;
+    /** DEPTH dial forwarded to the proposer's `propose()` as
+     *  `ctx.maxImprovementShots` — max iterations an agentic candidate generator
+     *  may take per candidate (verify-in-session retries). Unset ⇒ the
+     *  proposer's own default. */
+    maxImprovementShots?: number;
     /** @deprecated Must be 1 when supplied. The loop promotes only a candidate
      *  that replaces its single global incumbent. */
     promoteTopK?: number;
@@ -3801,7 +3834,7 @@ type SelfImproveProgressEvent = {
 } | {
     kind: 'gate.decided';
     decision: string;
-    lift: number;
+    lift?: number;
 } | {
     kind: 'power.estimated';
     n: number;
@@ -3835,6 +3868,18 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
     baselineSurface: MutableSurface;
     /** Budget + loop shape. All fields optional. */
     budget?: SelfImproveBudget;
+    /**
+     * Complete prior measurement of `baselineSurface` over the TRAIN split.
+     * Forwarded to the loop body, which validates its surface hash, scenario
+     * split, seed (42), reps, and coverage, then skips the baseline search
+     * campaign entirely — no baseline dispatch, no resumability lookup. The
+     * train split is `scenarios` minus the holdout split, so premeasure with
+     * exactly that scenario set (explicit `budget.holdoutScenarios`, or
+     * `budget.holdout: 'deferred'` with no reserved set, makes the train split
+     * deterministic). Prior spend stays in the imported campaign aggregates and
+     * is not re-added to this run's cost ledger.
+     */
+    premeasuredBaseline?: PremeasuredOptimizationBaseline<TArtifact, TScenario>;
     /** Custom surface proposer. Default is `gepaProposer` configured from `llm` +
      *  `mutationPrimitives`. */
     proposer?: SurfaceProposer;
@@ -3934,12 +3979,16 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
     findings?: unknown[];
 }
 interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
-    /** Composite mean across all scenarios, baseline run. */
+    /** Composite mean across all scenarios, baseline run. When
+     *  `budget.holdout === 'deferred'` this is measured on the improvement
+     *  (search) split — no holdout campaign ran. */
     baseline: {
         compositeMean: number;
         perScenario: Record<string, number>;
     };
-    /** Composite mean on the held-out set, winner run. */
+    /** Composite mean on the held-out set, winner run. When
+     *  `budget.holdout === 'deferred'` this is the winner's improvement-set
+     *  (search) measurement — no holdout campaign ran. */
     winner: {
         compositeMean: number;
         perScenario: Record<string, number>;
@@ -3953,8 +4002,10 @@ interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
         rationale?: string;
     };
     /** `winner.compositeMean - baselineOnHoldout.compositeMean`. Positive
-     *  means the gate observed improvement. */
-    lift: number;
+     *  means the gate observed improvement. Absent iff
+     *  `budget.holdout === 'deferred'` — no held-out measurement ran, so there
+     *  is no lift to report (never a fabricated 0). */
+    lift?: number;
     /** The explicit baseline→winner unified diff. Always present (empty string
      *  when winner == baseline). */
     diff: string;

package/dist/contract/index.js

@@ -36,7 +36,7 @@ import {
   runReferenceEquivalenceJudge,
   surfaceContentHash,
   surfaceHash
-} from "../chunk-S6IL3R5I.js";
+} from "../chunk-HZJF4IUO.js";
 import {
   campaignSplitDigest,
   createRunCostLedger,
@@ -48,9 +48,9 @@ import {
 import {
   buildDefaultAnalystRegistry,
   createChatClient
-} from "../chunk-BVMWQDC5.js";
+} from "../chunk-VJ7T5WIO.js";
 import "../chunk-HHWE3POT.js";
-import "../chunk-MUS5DOH7.js";
+import "../chunk-FC5NDO3E.js";
 import {
   FileSystemOutcomeStore,
   InMemoryOutcomeStore
@@ -70,7 +70,7 @@ import {
 import "../chunk-PXE2VKMX.js";
 import "../chunk-ZET2UAYW.js";
 import "../chunk-GGE4NNQT.js";
-import "../chunk-PXD6ZFNY.js";
+import "../chunk-IR3KBHOY.js";
 import "../chunk-PC4UYEBM.js";
 import {
   parseRunRecordSafe
@@ -125,6 +125,15 @@ function meanComposite(byScenario) {
     perScenario
   };
 }
+function winnerSearchCampaign(result) {
+  for (let i = result.generations.length - 1; i >= 0; i--) {
+    const measured = result.generations[i]?.surfaces.find(
+      (s) => s.surfaceHash === result.winnerSurfaceHash
+    );
+    if (measured) return measured.campaign;
+  }
+  return result.baselineCampaign;
+}
 async function selfImprove(opts) {
   const startedAt = Date.now();
   const requestedRunDir = opts.runDir ?? `mem://selfImprove-${startedAt}`;
@@ -147,18 +156,20 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
   const populationSize = budget.populationSize ?? 2;
   const maxConcurrency = budget.maxConcurrency ?? 2;
   const holdoutFraction = budget.holdoutFraction ?? 0.25;
+  const holdoutMode = budget.holdout ?? "measured";
+  const holdoutDeferred = holdoutMode === "deferred";
   const expectUsage = opts.expectUsage ?? "assert";
   const explicitHoldout = budget.holdoutScenarios;
   const { train, holdout } = explicitHoldout ? {
     train: opts.scenarios.filter((s) => !explicitHoldout.some((h) => h.id === s.id)),
     holdout: explicitHoldout
-  } : splitTrainHoldout(opts.scenarios, holdoutFraction);
+  } : holdoutDeferred ? { train: opts.scenarios, holdout: [] } : splitTrainHoldout(opts.scenarios, holdoutFraction);
   if (train.length === 0) {
     throw new Error(
       "selfImprove: train split is empty. Reduce holdoutFraction or pass more scenarios."
     );
   }
-  if (holdout.length === 0) {
+  if (holdout.length === 0 && !holdoutDeferred) {
     throw new Error("selfImprove: holdout split is empty. Pass more scenarios.");
   }
   const proposer = opts.proposer ?? gepaProposer({
@@ -182,6 +193,7 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
   const result = await runImprovementLoop({
     scenarios: train,
     baselineSurface: opts.baselineSurface,
+    premeasuredBaseline: opts.premeasuredBaseline,
     dispatchWithSurface: opts.agent,
     proposer,
     judges: [opts.judge],
@@ -189,7 +201,9 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
     maxGenerations: generations,
     promoteTopK: budget.promoteTopK,
     reps: budget.reps,
+    maxImprovementShots: budget.maxImprovementShots,
     holdoutScenarios: holdout,
+    holdout: holdoutMode,
     gate,
     neutralize: opts.neutralize,
     autoOnPromote: opts.autoOnPromote ?? "none",
@@ -207,8 +221,11 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
     analyzeGeneration: opts.analyzeGeneration,
     findings: opts.findings
   });
-  const baseline = meanComposite(result.baselineOnHoldout.aggregates.byScenario);
-  const winnerStats = meanComposite(result.winnerOnHoldout.aggregates.byScenario);
+  const winnerSearch = holdoutDeferred ? winnerSearchCampaign(result) : void 0;
+  const baseline = meanComposite(
+    (holdoutDeferred ? result.baselineCampaign : result.baselineOnHoldout).aggregates.byScenario
+  );
+  const winnerStats = meanComposite((winnerSearch ?? result.winnerOnHoldout).aggregates.byScenario);
   let power;
   const baselineHoldoutComposites = result.baselineOnHoldout.cells.filter((cell) => !cell.error).map((cell) => {
     const scores = Object.values(cell.judgeScores);
@@ -241,7 +258,10 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
     opts.onProgress({
       kind: "gate.decided",
       decision: result.gateResult.decision,
-      lift: winnerStats.compositeMean - baseline.compositeMean
+      // Deferred holdout has no held-out measurement: in that mode the summary
+      // stats are search-split numbers, and emitting their delta as `lift`
+      // would misreport a train-split delta as a held-out one. Omit instead.
+      ...holdoutDeferred ? {} : { lift: winnerStats.compositeMean - baseline.compositeMean }
     });
   }
   const cost = result.cost;
@@ -249,7 +269,12 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
   const insight = await analyzeRuns({
     runs: [
       ...cellsToRunRecords(result.baselineCampaign.cells, "baseline", runDir, opts.baselineSurface),
-      ...cellsToRunRecords(result.winnerOnHoldout.cells, "winner", runDir, result.winnerSurface)
+      ...cellsToRunRecords(
+        (winnerSearch ?? result.winnerOnHoldout).cells,
+        "winner",
+        runDir,
+        result.winnerSurface
+      )
     ],
     baselineCandidateId: "baseline",
     candidateCandidateId: "winner"
@@ -278,7 +303,7 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
       ...result.winnerLabel ? { label: result.winnerLabel } : {},
       ...result.winnerRationale ? { rationale: result.winnerRationale } : {}
     },
-    lift: winnerStats.compositeMean - baseline.compositeMean,
+    ...holdoutDeferred ? {} : { lift: winnerStats.compositeMean - baseline.compositeMean },
     diff: result.promotedDiff,
     provenance,
     gateDecision: result.gateResult.decision,