@tangle-network/agent-eval 0.126.0 → 0.126.1

package/dist/{chunk-KO2PZOGP.js → chunk-NGUYT5CI.js}

@@ -7,7 +7,7 @@ import {
   pairHoldout,
   recoverTruncatedJson,
   surfaceContentHash
-} from "./chunk-NTOV7RU5.js";
+} from "./chunk-VMUENW6F.js";
 import {
   SearchLedgerConflictError,
   SearchLedgerError,
@@ -22,7 +22,7 @@ import {
 } from "./chunk-UCLVDLCH.js";
 import {
   Mutex
-} from "./chunk-UI4YMIN2.js";
+} from "./chunk-WGXIEX7P.js";
 import {
   eProcess,
   mcnemar,
@@ -4634,4 +4634,4 @@ export {
   verifyCodeSurface,
   resolveWorktreePath
 };
-//# sourceMappingURL=chunk-KO2PZOGP.js.map
+//# sourceMappingURL=chunk-NGUYT5CI.js.map

package/dist/{chunk-NTOV7RU5.js → chunk-VMUENW6F.js}

@@ -16,8 +16,9 @@ import {
 } from "./chunk-UCLVDLCH.js";
 import {
   clamp01,
+  combineAbortSignals,
   mapConcurrent
-} from "./chunk-UI4YMIN2.js";
+} from "./chunk-WGXIEX7P.js";
 import {
   detectRewardHacking
 } from "./chunk-ARU2PZFM.js";
@@ -1696,7 +1697,6 @@ ${describe(winnerSurface)}`;
 }
 
 // src/campaign/presets/compare-optimization-methods.ts
-import { randomUUID } from "crypto";
 async function compareOptimizationMethods(opts) {
   assertOptimizationMethods(opts.methods);
   assertComparisonPartitions(opts);
@@ -1711,20 +1711,22 @@ async function compareOptimizationMethods(opts) {
   assertComparisonControls(opts, seed, resamples, confidence);
   const storage = opts.storage ?? fsCampaignStorage();
   const resolvedRunDir = resolveRunDir(opts.runDir, opts.repo);
-  const testCostPhase = `compareOptimizationMethods:test:${randomUUID()}`;
-  const testCostLedger = opts.costLedger ?? createRunCostLedger({
+  const baselineSurface = structuredClone(opts.baselineSurface);
+  const costLedger = opts.costLedger ?? createRunCostLedger({
     storage,
-    runDir: `${resolvedRunDir}/test/cost`,
+    runDir: `${resolvedRunDir}/cost`,
     costCeilingUsd: opts.costCeiling
   });
-  const scoreOnTest = async (surface, tag) => {
+  const scoreOnTest = async (surface, tag, costPhase) => {
+    const measuredSurface = structuredClone(surface);
     const campaign = await runCampaign({
       ...opts,
       storage,
-      costLedger: testCostLedger,
-      costPhase: testCostPhase,
+      costLedger,
+      costPhase,
       scenarios: opts.testScenarios.map((scenario) => structuredClone(scenario)),
-      dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
+      dispatch: (scenario, ctx) => opts.dispatchWithSurface(structuredClone(measuredSurface), scenario, ctx),
+      dispatchRef: finalDispatchRef(opts, measuredSurface),
       runDir: `${resolvedRunDir}/${tag}`
     });
     const byScenario = {};
@@ -1745,22 +1747,40 @@ async function compareOptimizationMethods(opts) {
     }
     return scenarioIds.map((id) => byScenario[id]);
   };
+  const optimizationOwner = new AbortController();
   const optimized = await mapConcurrent(opts.methods, optimizationConcurrency, async (method) => {
-    const out = await method.optimize(
-      createOptimizationMethodInput(opts, method.name, resolvedRunDir, seed)
-    );
-    assertOptimizationResult(method.name, out);
-    const winnerSurface = structuredClone(out.winnerSurface);
-    return {
-      name: method.name,
-      winnerSurface,
-      cost: out.cost,
-      durationMs: out.durationMs,
-      provenance: out.provenance
-    };
+    try {
+      const out = await method.optimize(
+        createOptimizationMethodInput(
+          opts,
+          method.name,
+          resolvedRunDir,
+          seed,
+          baselineSurface,
+          costLedger,
+          optimizationOwner.signal
+        )
+      );
+      assertOptimizationResult(method.name, out);
+      const winnerSurface = structuredClone(out.winnerSurface);
+      return {
+        name: method.name,
+        winnerSurface,
+        cost: out.cost,
+        durationMs: out.durationMs,
+        provenance: out.provenance
+      };
+    } catch (error) {
+      if (!optimizationOwner.signal.aborted) optimizationOwner.abort(error);
+      throw error;
+    }
   });
-  const baselineArr = align(await scoreOnTest(opts.baselineSurface, "test/baseline"), "baseline");
-  const testScoresBySurface = /* @__PURE__ */ new Map([[surfaceContentHash(opts.baselineSurface), baselineArr]]);
+  const testCostPhase = finalCostPhase(opts, baselineSurface, optimized, seed);
+  const baselineArr = align(
+    await scoreOnTest(baselineSurface, "test/baseline", testCostPhase),
+    "baseline"
+  );
+  const testScoresBySurface = /* @__PURE__ */ new Map([[surfaceContentHash(baselineSurface), baselineArr]]);
   const winners = [];
   for (const winner of optimized) {
     const surfaceKey = surfaceContentHash(winner.winnerSurface);
@@ -1768,7 +1788,8 @@ async function compareOptimizationMethods(opts) {
     if (!arr) {
       const byScenario = await scoreOnTest(
         winner.winnerSurface,
-        `test/methods/${slug(winner.name)}`
+        `test/methods/${slug(winner.name)}`,
+        testCostPhase
       );
       arr = align(byScenario, `method "${winner.name}"`);
       testScoresBySurface.set(surfaceKey, arr);
@@ -1798,7 +1819,7 @@ async function compareOptimizationMethods(opts) {
         winnerComposite: w.arr[index],
         lift: w.arr[index] - baselineArr[index]
       })),
-      winnerSurface: w.winnerSurface,
+      winnerSurface: structuredClone(w.winnerSurface),
       rank: 0
     };
     if (w.durationMs !== void 0) score.durationMs = w.durationMs;
@@ -1843,7 +1864,7 @@ async function compareOptimizationMethods(opts) {
   const optimizationCost = combineCosts(
     scores.map((score) => ({ label: `method '${score.name}'`, cost: score.optimizationCost }))
   );
-  const testCost = costFromLedgerSummary(testCostLedger.summary({ phase: testCostPhase }));
+  const testCost = costFromLedgerSummary(costLedger.summary({ phase: testCostPhase }));
   const totalCost = combineCosts([
     { label: "optimization", cost: optimizationCost },
     { label: "final test", cost: testCost }
@@ -1961,12 +1982,22 @@ function assertOptimizationProvenance(methodName, value) {
   }
 }
 function assertComparisonControls(opts, seed, resamples, confidence) {
+  if (opts.optimizationRunOptions && "costCeiling" in opts.optimizationRunOptions) {
+    throw new Error(
+      "compareOptimizationMethods: optimizationRunOptions.costCeiling is not supported; costCeiling covers optimization and final scoring"
+    );
+  }
   if (!opts.judges || opts.judges.length === 0) {
     throw new Error("compareOptimizationMethods: at least one judge is required");
   }
   if (typeof opts.dispatchWithSurface !== "function") {
     throw new Error("compareOptimizationMethods: dispatchWithSurface must be a function");
   }
+  if (opts.dispatchRef !== void 0 && (typeof opts.dispatchRef !== "string" || opts.dispatchRef.trim().length === 0 || opts.dispatchRef.trim() !== opts.dispatchRef)) {
+    throw new Error(
+      "compareOptimizationMethods: dispatchRef must be trimmed and non-empty when provided"
+    );
+  }
   try {
     surfaceContentHash(opts.baselineSurface);
   } catch (cause) {
@@ -2112,9 +2143,8 @@ function minimumBootstrapResamples(confidence, comparisonCount) {
   const exact = 2 * comparisonCount / (1 - confidence);
   return Math.ceil(exact - Number.EPSILON * Math.max(1, exact) * 32);
 }
-function createOptimizationMethodInput(opts, methodName, resolvedRunDir, seed) {
+function createOptimizationMethodInput(opts, methodName, resolvedRunDir, seed, baselineSurface, costLedger, optimizationSignal) {
   const methodRunDir = `${resolvedRunDir}/optimization/${slug(methodName)}`;
-  const storage = opts.storage ?? fsCampaignStorage();
   const cloneScenarios = (scenarios) => Object.freeze(scenarios.map((scenario) => structuredClone(scenario)));
   const judges = opts.judges.map(
     (judge) => Object.freeze({
@@ -2124,21 +2154,53 @@ function createOptimizationMethodInput(opts, methodName, resolvedRunDir, seed) {
       )
     })
   );
+  const signal = combineAbortSignals(
+    opts.signal,
+    opts.optimizationRunOptions?.signal,
+    optimizationSignal
+  );
   return Object.freeze({
-    baselineSurface: structuredClone(opts.baselineSurface),
+    baselineSurface: structuredClone(baselineSurface),
     trainScenarios: cloneScenarios(opts.trainScenarios),
     selectionScenarios: cloneScenarios(opts.selectionScenarios),
     dispatchWithSurface: opts.dispatchWithSurface,
     judges: Object.freeze(judges),
     runDir: methodRunDir,
     seed,
-    runOptions: Object.freeze({ ...opts.optimizationRunOptions ?? {} }),
-    costLedger: createRunCostLedger({
-      storage,
-      runDir: `${methodRunDir}/cost`,
-      costCeilingUsd: opts.optimizationRunOptions?.costCeiling
-    })
+    runOptions: Object.freeze({
+      ...opts.optimizationRunOptions ?? {},
+      ...signal ? { signal } : {}
+    }),
+    costLedger
+  });
+}
+function finalCostPhase(opts, baselineSurface, winners, seed) {
+  const identity = contentHash({
+    baseline: surfaceContentHash(baselineSurface),
+    winners: winners.map((winner) => ({
+      name: winner.name,
+      surface: surfaceContentHash(winner.winnerSurface)
+    })),
+    testScenarios: opts.testScenarios,
+    judges: opts.judges.map((judge) => ({
+      name: judge.name,
+      dimensions: judge.dimensions,
+      version: judge.judgeVersion ?? contentHash({
+        score: judge.score.toString(),
+        appliesTo: judge.appliesTo?.toString() ?? null
+      })
+    })),
+    dispatch: callerDispatchRef(opts),
+    seed,
+    reps: opts.reps ?? 1
   });
+  return `compareOptimizationMethods.test:${identity}`;
+}
+function finalDispatchRef(opts, surface) {
+  return `compareOptimizationMethods:${callerDispatchRef(opts)}:${surfaceContentHash(surface)}`;
+}
+function callerDispatchRef(opts) {
+  return (opts.dispatchRef ?? opts.dispatchWithSurface.name) || "anonymous";
 }
 function costFromLedgerSummary(summary) {
   const cost = {
@@ -2384,6 +2446,7 @@ function sendJson(response, status, body) {
 var MAX_CALLBACK_BODY_BYTES = 1e6;
 async function startExternalOptimizerCallback(args) {
   assertCallbackConfig(args);
+  args.signal?.throwIfAborted();
   let evaluations = 0;
   let accepting = true;
   let closePromise;
@@ -2397,6 +2460,7 @@ async function startExternalOptimizerCallback(args) {
     const controller = new AbortController();
     const abortRequest = () => {
       request.destroy();
+      response.destroy();
     };
     activeControllers.add(controller);
     controller.signal.addEventListener("abort", abortRequest, { once: true });
@@ -2419,16 +2483,23 @@ async function startExternalOptimizerCallback(args) {
     void handler.catch(() => void 0);
   });
   const port = await listenLocal(server);
+  const close = () => {
+    closePromise ??= closeCallbackServer();
+    return closePromise;
+  };
+  const onAbort = () => {
+    void close().catch(() => void 0);
+  };
+  args.signal?.addEventListener("abort", onAbort, { once: true });
+  if (args.signal?.aborted) onAbort();
   return {
     url: `http://127.0.0.1:${port}/evaluate`,
     token: args.token,
     evaluations: () => evaluations,
-    close: () => {
-      closePromise ??= closeCallbackServer();
-      return closePromise;
-    }
+    close
   };
   async function closeCallbackServer() {
+    args.signal?.removeEventListener("abort", onAbort);
     accepting = false;
     const closingServer = closeServer(server);
     server.closeIdleConnections?.();
@@ -2525,6 +2596,7 @@ import { createServer as createServer2 } from "http";
 var MODEL_PROXY_PATHS = /* @__PURE__ */ new Set(["/v1/chat/completions", "/v1/responses"]);
 async function startExternalOptimizerModelProxy(args) {
   assertModelProxyConfig(args);
+  args.signal?.throwIfAborted();
   const token = randomLocalToken();
   const fetchImpl = args.fetchImpl ?? fetch;
   let requestCount = 0;
@@ -2544,6 +2616,7 @@ async function startExternalOptimizerModelProxy(args) {
     const controller = new AbortController();
     const abortRequest = () => {
       request.destroy();
+      response.destroy();
     };
     activeControllers.add(controller);
     controller.signal.addEventListener("abort", abortRequest, { once: true });
@@ -2583,17 +2656,24 @@ async function startExternalOptimizerModelProxy(args) {
     void handler.catch(() => void 0);
   });
   const port = await listenLocal(server);
+  const close = () => {
+    closePromise ??= closeModelProxy();
+    return closePromise;
+  };
+  const onAbort = () => {
+    void close().catch(() => void 0);
+  };
+  args.signal?.addEventListener("abort", onAbort, { once: true });
+  if (args.signal?.aborted) onAbort();
   return {
     baseUrl: `http://127.0.0.1:${port}/v1`,
     apiKey: token,
     requestAttempts: () => requestCount,
     successfulCompletions: () => successfulCompletionCount,
-    close: () => {
-      closePromise ??= closeModelProxy();
-      return closePromise;
-    }
+    close
   };
   async function closeModelProxy() {
+    args.signal?.removeEventListener("abort", onAbort);
     accepting = false;
     const closingServer = closeServer(server);
     server.closeIdleConnections?.();
@@ -3041,6 +3121,7 @@ async function runExternalOptimizerProcess(args) {
       `${args.label} requires POSIX process-group cleanup; run the optimizer through WSL or Linux`
     );
   }
+  throwIfAborted(args.signal, args.label);
   const dir = await mkdtemp(join2(tmpdir2(), args.tempPrefix));
   const inputPath = join2(dir, "input.json");
   const outputPath = join2(dir, "output.json");
@@ -3074,11 +3155,14 @@ async function runExternalOptimizerProcess(args) {
         args: commandArgs,
         cwd: dir,
         env: args.runner?.env,
-        timeoutMs: args.timeoutMs
+        timeoutMs: args.timeoutMs,
+        signal: args.signal
       });
+      throwIfAborted(args.signal, args.label);
       const raw = JSON.parse(
         await readBoundedTextFile(outputPath, MAX_PROCESS_RESULT_BYTES, `${args.label} output`)
       );
+      throwIfAborted(args.signal, args.label);
       if (!isRecord(raw)) throw new Error(`${args.label} output must be a JSON object`);
       return raw;
     },
@@ -3106,6 +3190,7 @@ async function readBoundedTextFile(path, maxBytes, label) {
   }
 }
 function runProcess(args) {
+  throwIfAborted(args.signal, args.label);
   return new Promise((resolvePromise, reject) => {
     const child = spawn(args.command, args.args, {
       cwd: args.cwd,
@@ -3118,10 +3203,12 @@ function runProcess(args) {
     const stderr = createProcessOutputCapture();
     let settled = false;
     let timeout;
+    const onAbort = () => finish(abortReason(args.signal, args.label));
     const finish = (error) => {
       if (settled) return;
       settled = true;
       if (timeout) clearTimeout(timeout);
+      args.signal?.removeEventListener("abort", onAbort);
       void terminateProcessTree(child).then(
         () => {
           if (error) reject(error);
@@ -3149,6 +3236,8 @@ function runProcess(args) {
     timeout = setTimeout(() => {
       finish(new Error(`${args.label} exceeded ${args.timeoutMs}ms`));
     }, args.timeoutMs);
+    args.signal?.addEventListener("abort", onAbort, { once: true });
+    if (args.signal?.aborted) onAbort();
     child.stdout.on("data", (chunk) => {
       appendProcessOutput(stdout, chunk);
     });
@@ -3171,6 +3260,12 @@ function runProcess(args) {
     });
   });
 }
+function throwIfAborted(signal, label) {
+  if (signal?.aborted) throw abortReason(signal, label);
+}
+function abortReason(signal, label) {
+  return signal.reason instanceof Error ? signal.reason : new Error(`${label} aborted`, { cause: signal.reason });
+}
 function safeProcessEnvironment() {
   const allowed = [
     "PATH",
@@ -4912,7 +5007,8 @@ async function inspectExternalOptimizerRuntime(args) {
       engineModules: [...args.engineModules ?? []]
     },
     ...args.runner ? { runner: args.runner } : {},
-    timeoutMs: args.timeoutMs
+    timeoutMs: args.timeoutMs,
+    ...args.signal ? { signal: args.signal } : {}
   });
   assertExternalOptimizerRuntimeIdentity(result.runtime, args.package, args.label);
   return result.runtime;
@@ -5311,6 +5407,8 @@ function gepaOptimizationMethod(config) {
   return {
     name,
     async optimize(input) {
+      const signal = input.runOptions.signal;
+      signal?.throwIfAborted();
       if (typeof input.baselineSurface !== "string" && input.baselineSurface.kind !== "components") {
         throw new Error(`${name}: GEPA bridge supports text and component surfaces`);
       }
@@ -5337,7 +5435,8 @@ function gepaOptimizationMethod(config) {
         module: "agent_eval_rpc.gepa_bridge",
         engineModules: config.engineModules,
         ...bridgeRunner ? { runner: bridgeRunner } : {},
-        timeoutMs: config.timeoutMs ?? GEPA_DEFAULT_TIMEOUT_MS
+        timeoutMs: config.timeoutMs ?? GEPA_DEFAULT_TIMEOUT_MS,
+        ...signal ? { signal } : {}
       });
       const seedCandidate = encodeExternalTextCandidate(input.baselineSurface);
       const trainSet = input.trainScenarios.map(
@@ -5408,7 +5507,8 @@ function gepaOptimizationMethod(config) {
         token: randomBytes3(32).toString("hex"),
         maxEvaluations: evaluationLimit,
         acceptEvaluation: () => runBudget.acceptEvaluation(),
-        evaluate
+        evaluate,
+        ...signal ? { signal } : {}
       });
       const runnerEnv = bridgeRunner?.env ?? {};
       let modelProxy;
@@ -5438,7 +5538,8 @@ function gepaOptimizationMethod(config) {
               initialUsage: {
                 requests: priorOptimizerUsage.totalCalls,
                 costUsd: priorOptimizerUsage.totalCostUsd
-              }
+              },
+              ...signal ? { signal } : {}
             });
           }
           const outputDir2 = `${runDir}/external`;
@@ -5481,12 +5582,14 @@ function gepaOptimizationMethod(config) {
               ...bridgeRunner,
               env: removeCredentialEnvironment(runnerEnv)
             } : bridgeRunner,
-            timeoutMs: config.timeoutMs ?? GEPA_DEFAULT_TIMEOUT_MS
+            timeoutMs: config.timeoutMs ?? GEPA_DEFAULT_TIMEOUT_MS,
+            ...signal ? { signal } : {}
           });
           return { result: result2, outputDir: outputDir2 };
         },
         cleanup: closeResources
       });
+      signal?.throwIfAborted();
       assertGepaBridgeOutput(
         result,
         name,
@@ -6840,6 +6943,8 @@ function skillOptOptimizationMethod(config) {
   return {
     name,
     async optimize(input) {
+      const signal = input.runOptions.signal;
+      signal?.throwIfAborted();
       if (typeof input.baselineSurface !== "string") {
         throw new Error(`${name}: SkillOpt requires a string baselineSurface`);
       }
@@ -6861,7 +6966,8 @@ function skillOptOptimizationMethod(config) {
         package: "skillopt",
         module: "agent_eval_rpc.skillopt_bridge",
         ...bridgeRunner ? { runner: bridgeRunner } : {},
-        timeoutMs: config.timeoutMs ?? SKILLOPT_DEFAULT_TIMEOUT_MS
+        timeoutMs: config.timeoutMs ?? SKILLOPT_DEFAULT_TIMEOUT_MS,
+        ...signal ? { signal } : {}
       });
       const trainSet = input.trainScenarios.map(
         (scenario) => describeExternalScenario(scenario, "SkillOpt", maxEvidenceChars, config.describeScenario)
@@ -6927,7 +7033,8 @@ function skillOptOptimizationMethod(config) {
         token: randomBytes4(32).toString("hex"),
         maxEvaluations: config.maxEvaluations,
         acceptEvaluation: () => runBudget.acceptEvaluation(),
-        evaluate
+        evaluate,
+        ...signal ? { signal } : {}
       });
       const runnerEnv = bridgeRunner?.env ?? {};
       let activeModelProxy;
@@ -6956,7 +7063,8 @@ function skillOptOptimizationMethod(config) {
             initialUsage: {
               requests: priorOptimizerUsage.totalCalls,
               costUsd: priorOptimizerUsage.totalCostUsd
-            }
+            },
+            ...signal ? { signal } : {}
           });
           activeModelProxy = modelProxy2;
           const outputDir2 = `${runDir}/external`;
@@ -6998,15 +7106,29 @@ function skillOptOptimizationMethod(config) {
                 OPTIMIZER_OPENAI_COMPATIBLE_BASE_URL: modelProxy2.baseUrl,
                 OPTIMIZER_OPENAI_COMPATIBLE_API_KEY: modelProxy2.apiKey,
                 TARGET_OPENAI_COMPATIBLE_BASE_URL: modelProxy2.baseUrl,
-                TARGET_OPENAI_COMPATIBLE_API_KEY: modelProxy2.apiKey
+                TARGET_OPENAI_COMPATIBLE_API_KEY: modelProxy2.apiKey,
+                OPENAI_COMPATIBLE_MODEL: config.optimizer.model,
+                OPENAI_COMPATIBLE_MAX_TOKENS: String(
+                  config.optimizer.budget.maxOutputTokensPerRequest
+                ),
+                OPTIMIZER_OPENAI_COMPATIBLE_MODEL: config.optimizer.model,
+                OPTIMIZER_OPENAI_COMPATIBLE_MAX_TOKENS: String(
+                  config.optimizer.budget.maxOutputTokensPerRequest
+                ),
+                TARGET_OPENAI_COMPATIBLE_MODEL: config.optimizer.model,
+                TARGET_OPENAI_COMPATIBLE_MAX_TOKENS: String(
+                  config.optimizer.budget.maxOutputTokensPerRequest
+                )
               }
             },
-            timeoutMs: config.timeoutMs ?? SKILLOPT_DEFAULT_TIMEOUT_MS
+            timeoutMs: config.timeoutMs ?? SKILLOPT_DEFAULT_TIMEOUT_MS,
+            ...signal ? { signal } : {}
           });
           return { result: result2, outputDir: outputDir2, modelProxy: modelProxy2 };
         },
         cleanup: closeResources
       });
+      signal?.throwIfAborted();
       assertSkillOptBridgeOutput(result, name, maxCandidateChars, config.maxEvaluations);
       assertExternalOptimizerRunBinding({
         label: name,
@@ -7149,4 +7271,4 @@ export {
   emitLoopProvenance,
   skillOptOptimizationMethod
 };
-//# sourceMappingURL=chunk-NTOV7RU5.js.map
+//# sourceMappingURL=chunk-VMUENW6F.js.map