@tangle-network/agent-eval 0.126.0 → 0.126.1

package/dist/{chunk-UI4YMIN2.js → chunk-WGXIEX7P.js}

@@ -95,11 +95,22 @@ function finiteOrZero(value) {
   return Number.isFinite(value) ? value : 0;
 }
 
+// src/abort-signal.ts
+function combineAbortSignals(...signals) {
+  const active = [
+    ...new Set(signals.filter((signal) => signal !== void 0))
+  ];
+  if (active.length === 0) return void 0;
+  if (active.length === 1) return active[0];
+  return AbortSignal.any(active);
+}
+
 export {
+  combineAbortSignals,
   Mutex,
   mapConcurrent,
   DEFAULT_RUN_SCORE_WEIGHTS,
   aggregateRunScore,
   clamp01
 };
-//# sourceMappingURL=chunk-UI4YMIN2.js.map
+//# sourceMappingURL=chunk-WGXIEX7P.js.map

package/dist/chunk-WGXIEX7P.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/concurrency.ts","../src/run-score.ts","../src/abort-signal.ts"],"sourcesContent":["/**\n * concurrency — small primitives the evolution loop needs.\n *\n * `Mutex` is a zero-dep async lock with FIFO fairness. The evolution loop\n * uses it to serialise checkout/build/commit sequences inside a single\n * pool slot, and to gate concurrent JSONL writers (see\n * `lockedJsonlReferenceReplayStore`).\n *\n * Deliberately minimal — no priority queue, no timeouts. If you need\n * those, swap to `async-mutex` at the call site.\n */\n\nexport class Mutex {\n  private locked = false\n  private readonly waiters: Array<() => void> = []\n\n  async acquire(): Promise<() => void> {\n    if (!this.locked) {\n      this.locked = true\n      return () => this.release()\n    }\n    return new Promise<() => void>((resolve) => {\n      this.waiters.push(() => {\n        resolve(() => this.release())\n      })\n    })\n  }\n\n  private release(): void {\n    const next = this.waiters.shift()\n    if (next) {\n      next()\n    } else {\n      this.locked = false\n    }\n  }\n\n  async runExclusive<T>(fn: () => Promise<T> | T): Promise<T> {\n    const release = await this.acquire()\n    try {\n      return await fn()\n    } finally {\n      release()\n    }\n  }\n\n  /** True iff someone holds the lock right now. Diagnostics only. */\n  get isLocked(): boolean {\n    return this.locked\n  }\n\n  /** Pending waiter count. Diagnostics only. */\n  get pending(): number {\n    return this.waiters.length\n  }\n}\n\n/**\n * Map independent work with a fixed worker count while preserving input order.\n * After the first rejection, no new items start; already-running work is allowed\n * to settle before the returned promise rejects. Partial results are discarded.\n */\nexport async function mapConcurrent<T, R>(\n  items: readonly T[],\n  concurrency: number,\n  map: (item: T, index: number) => Promise<R>,\n): Promise<R[]> {\n  if (!Number.isInteger(concurrency) || concurrency < 1) {\n    throw new Error(`mapConcurrent: concurrency must be a positive integer, got ${concurrency}`)\n  }\n  if (items.length === 0) return []\n\n  const results = new Array<R>(items.length)\n  let nextIndex = 0\n  let stopped = false\n  let failed = false\n  let failure: unknown\n\n  const worker = async (): Promise<void> => {\n    while (!stopped) {\n      const index = nextIndex\n      nextIndex += 1\n      if (index >= items.length) return\n\n      try {\n        results[index] = await map(items[index]!, index)\n      } catch (error) {\n        stopped = true\n        if (!failed) {\n          failed = true\n          failure = error\n        }\n      }\n    }\n  }\n\n  await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()))\n  if (failed) throw failure\n  return results\n}\n","export interface RunScore {\n  success: number\n  goalProgress: number\n  repoGroundedness: number\n  driftPenalty: number\n  toolUseQuality: number\n  patchQuality: number\n  testReality: number\n  finalGate: number\n  reviewerBlockers: number\n  costUsd: number\n  wallSeconds: number\n  notes?: string[]\n}\n\nexport interface RunScoreWeights {\n  success: number\n  goalProgress: number\n  repoGroundedness: number\n  driftPenalty: number\n  toolUseQuality: number\n  patchQuality: number\n  testReality: number\n  finalGate: number\n  reviewerBlockers: number\n  costUsd: number\n  wallSeconds: number\n}\n\nexport const DEFAULT_RUN_SCORE_WEIGHTS: RunScoreWeights = {\n  success: 4,\n  goalProgress: 2,\n  repoGroundedness: 1.5,\n  driftPenalty: -1.5,\n  toolUseQuality: 1,\n  patchQuality: 1.25,\n  testReality: 1.5,\n  finalGate: 3,\n  reviewerBlockers: -2,\n  costUsd: -0.2,\n  wallSeconds: -0.1,\n}\n\nexport function aggregateRunScore(score: RunScore, weights: Partial<RunScoreWeights> = {}): number {\n  const w = { ...DEFAULT_RUN_SCORE_WEIGHTS, ...weights }\n  return (\n    w.success * clamp01(score.success) +\n    w.goalProgress * clamp01(score.goalProgress) +\n    w.repoGroundedness * clamp01(score.repoGroundedness) +\n    w.driftPenalty * clamp01(score.driftPenalty) +\n    w.toolUseQuality * clamp01(score.toolUseQuality) +\n    w.patchQuality * clamp01(score.patchQuality) +\n    w.testReality * clamp01(score.testReality) +\n    w.finalGate * clamp01(score.finalGate) +\n    w.reviewerBlockers * clamp01(score.reviewerBlockers) +\n    w.costUsd * Math.max(0, finiteOrZero(score.costUsd)) +\n    w.wallSeconds * Math.max(0, finiteOrZero(score.wallSeconds) / 60)\n  )\n}\n\nexport function clamp01(value: number): number {\n  if (!Number.isFinite(value)) return 0\n  return Math.max(0, Math.min(1, value))\n}\n\nfunction finiteOrZero(value: number): number {\n  return Number.isFinite(value) ? value : 0\n}\n","/** Combine active cancellation sources without wrapping a single source. */\nexport function combineAbortSignals(\n  ...signals: Array<AbortSignal | undefined>\n): AbortSignal | undefined {\n  const active = [\n    ...new Set(signals.filter((signal): signal is AbortSignal => signal !== undefined)),\n  ]\n  if (active.length === 0) return undefined\n  if (active.length === 1) return active[0]\n  return AbortSignal.any(active)\n}\n"],"mappings":";AAYO,IAAM,QAAN,MAAY;AAAA,EACT,SAAS;AAAA,EACA,UAA6B,CAAC;AAAA,EAE/C,MAAM,UAA+B;AACnC,QAAI,CAAC,KAAK,QAAQ;AAChB,WAAK,SAAS;AACd,aAAO,MAAM,KAAK,QAAQ;AAAA,IAC5B;AACA,WAAO,IAAI,QAAoB,CAAC,YAAY;AAC1C,WAAK,QAAQ,KAAK,MAAM;AACtB,gBAAQ,MAAM,KAAK,QAAQ,CAAC;AAAA,MAC9B,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEQ,UAAgB;AACtB,UAAM,OAAO,KAAK,QAAQ,MAAM;AAChC,QAAI,MAAM;AACR,WAAK;AAAA,IACP,OAAO;AACL,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEA,MAAM,aAAgB,IAAsC;AAC1D,UAAM,UAAU,MAAM,KAAK,QAAQ;AACnC,QAAI;AACF,aAAO,MAAM,GAAG;AAAA,IAClB,UAAE;AACA,cAAQ;AAAA,IACV;AAAA,EACF;AAAA;AAAA,EAGA,IAAI,WAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,UAAkB;AACpB,WAAO,KAAK,QAAQ;AAAA,EACtB;AACF;AAOA,eAAsB,cACpB,OACA,aACA,KACc;AACd,MAAI,CAAC,OAAO,UAAU,WAAW,KAAK,cAAc,GAAG;AACrD,UAAM,IAAI,MAAM,8DAA8D,WAAW,EAAE;AAAA,EAC7F;AACA,MAAI,MAAM,WAAW,EAAG,QAAO,CAAC;AAEhC,QAAM,UAAU,IAAI,MAAS,MAAM,MAAM;AACzC,MAAI,YAAY;AAChB,MAAI,UAAU;AACd,MAAI,SAAS;AACb,MAAI;AAEJ,QAAM,SAAS,YAA2B;AACxC,WAAO,CAAC,SAAS;AACf,YAAM,QAAQ;AACd,mBAAa;AACb,UAAI,SAAS,MAAM,OAAQ;AAE3B,UAAI;AACF,gBAAQ,KAAK,IAAI,MAAM,IAAI,MAAM,KAAK,GAAI,KAAK;AAAA,MACjD,SAAS,OAAO;AACd,kBAAU;AACV,YAAI,CAAC,QAAQ;AACX,mBAAS;AACT,oBAAU;AAAA,QACZ;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,QAAQ,IAAI,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;AAC7F,MAAI,OAAQ,OAAM;AAClB,SAAO;AACT;;;ACtEO,IAAM,4BAA6C;AAAA,EACxD,SAAS;AAAA,EACT,cAAc;AAAA,EACd,kBAAkB;AAAA,EAClB,cAAc;AAAA,EACd,gBAAgB;AAAA,EAChB,cAAc;AAAA,EACd,aAAa;AAAA,EACb,WAAW;AAAA,EACX,kBAAkB;AAAA,EAClB,SAAS;AAAA,EACT,aAAa;AACf;AAEO,SAAS,kBAAkB,OAAiB,UAAoC,CAAC,GAAW;AACjG,QAAM,IAAI,EAAE,GAAG,2BAA2B,GAAG,QAAQ;AACrD,SACE,EAAE,UAAU,QAAQ,MAAM,OAAO,IACjC,EAAE,eAAe,QAAQ,MAAM,YAAY,IAC3C,EAAE,mBAAmB,QAAQ,MAAM,gBAAgB,IACnD,EAAE,eAAe,QAAQ,MAAM,YAAY,IAC3C,EAAE,iBAAiB,QAAQ,MAAM,cAAc,IAC/C,EAAE,eAAe,QAAQ,MAAM,YAAY,IAC3C,EAAE,cAAc,QAAQ,MAAM,WAAW,IACzC,EAAE,YAAY,QAAQ,MAAM,SAAS,IACrC,EAAE,mBAAmB,QAAQ,MAAM,gBAAgB,IACnD,EAAE,UAAU,KAAK,IAAI,GAAG,aAAa,MAAM,OAAO,CAAC,IACnD,EAAE,cAAc,KAAK,IAAI,GAAG,aAAa,MAAM,WAAW,IAAI,EAAE;AAEpE;AAEO,SAAS,QAAQ,OAAuB;AAC7C,MAAI,CAAC,OAAO,SAAS,KAAK,EAAG,QAAO;AACpC,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;AAEA,SAAS,aAAa,OAAuB;AAC3C,SAAO,OAAO,SAAS,KAAK,IAAI,QAAQ;AAC1C;;;AClEO,SAAS,uBACX,SACsB;AACzB,QAAM,SAAS;AAAA,IACb,GAAG,IAAI,IAAI,QAAQ,OAAO,CAAC,WAAkC,WAAW,MAAS,CAAC;AAAA,EACpF;AACA,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,MAAI,OAAO,WAAW,EAAG,QAAO,OAAO,CAAC;AACxC,SAAO,YAAY,IAAI,MAAM;AAC/B;","names":[]}

package/dist/contract/index.d.ts

@@ -1816,8 +1816,8 @@ interface ExternalOptimizerModelBudget {
  * data and compared with paired confidence intervals.
  */
 
-/** Per-method campaign settings. Each method receives its own spend account. */
-type OptimizationMethodRunOptions<TScenario extends Scenario$1, TArtifact> = Omit<RunCampaignOptions<TScenario, TArtifact>, 'costLedger' | 'dispatch' | 'judges' | 'runDir' | 'scenarios' | 'seed'>;
+/** Shared campaign settings applied to every optimization method. */
+type OptimizationMethodRunOptions<TScenario extends Scenario$1, TArtifact> = Omit<RunCampaignOptions<TScenario, TArtifact>, 'costCeiling' | 'costLedger' | 'dispatch' | 'judges' | 'runDir' | 'scenarios' | 'seed'>;
 /** Cost reported by a method or by final test scoring. */
 interface ComparisonCost {
     totalCostUsd: number;
@@ -1890,7 +1890,7 @@ interface OptimizationMethodInput<TScenario extends Scenario$1, TArtifact> {
     readonly seed: number;
     /** Shared defaults for every method. A method may override them explicitly. */
     readonly runOptions: Readonly<OptimizationMethodRunOptions<TScenario, TArtifact>>;
-    /** Durable spend account shared by the method's model and evaluation calls. */
+    /** Durable spend account shared by every method and final scoring. */
     readonly costLedger: CostLedgerHandle;
 }
 interface OptimizationMethodResult {
@@ -1999,8 +1999,7 @@ interface CompareOptimizationMethodsOptions<TScenario extends Scenario$1, TArtif
     /** Simultaneous confidence across method-vs-baseline and method-vs-method contrasts.
      *  Each bootstrap interval is Bonferroni-adjusted. Default 0.95. */
     confidence?: number;
-    /** Shared spend limit across baseline and winner scoring on the final test partition.
-     *  Each method owns its optimization budget through `optimizationRunOptions.costCeiling`. */
+    /** Shared spend limit across every method's optimizer and evaluation calls plus final scoring. */
     costCeiling?: number;
 }
 /**

package/dist/contract/index.js

@@ -40,7 +40,7 @@ import {
   skillOptOptimizationMethod,
   surfaceContentHash,
   surfaceHash
-} from "../chunk-NTOV7RU5.js";
+} from "../chunk-VMUENW6F.js";
 import {
   campaignSplitDigest,
   createRunCostLedger,
@@ -52,9 +52,9 @@ import {
 import {
   buildDefaultAnalystRegistry,
   createChatClient
-} from "../chunk-CM4OILD2.js";
+} from "../chunk-LUNF2SEL.js";
 import "../chunk-HHWE3POT.js";
-import "../chunk-UI4YMIN2.js";
+import "../chunk-WGXIEX7P.js";
 import {
   FileSystemOutcomeStore,
   InMemoryOutcomeStore

package/dist/index.js

@@ -87,7 +87,7 @@ import {
   pairArms,
   parseCorrectnessResponse,
   verifyCompletion
-} from "./chunk-KO2PZOGP.js";
+} from "./chunk-NGUYT5CI.js";
 import {
   DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
@@ -121,7 +121,7 @@ import {
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-NTOV7RU5.js";
+} from "./chunk-VMUENW6F.js";
 import {
   BackendIntegrityError,
   assertRealAgentReceipts,
@@ -146,7 +146,7 @@ import {
   defaultIsMaterial,
   diffFindings,
   runSemanticConceptJudge
-} from "./chunk-4B7ZZHPX.js";
+} from "./chunk-KE2VWPZX.js";
 import {
   AnalystRegistry,
   DEFAULT_TRACE_ANALYST_KINDS,
@@ -163,7 +163,7 @@ import {
   makeFinding,
   renderPriorFindings,
   renderUpstreamFindings
-} from "./chunk-CM4OILD2.js";
+} from "./chunk-LUNF2SEL.js";
 import "./chunk-HHWE3POT.js";
 import {
   DEFAULT_RUN_SCORE_WEIGHTS,
@@ -171,7 +171,7 @@ import {
   aggregateRunScore,
   clamp01,
   mapConcurrent
-} from "./chunk-UI4YMIN2.js";
+} from "./chunk-WGXIEX7P.js";
 import {
   allCriticalPassed,
   controlFailureClassFromVerification,

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.126.0",
+    "version": "0.126.1",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/docs/campaign-proposers.md

@@ -183,13 +183,13 @@ const comparison = await compareOptimizationMethods({
   runDir: '.agent-eval/optimizer-comparison',
   optimizationRunOptions: {
     maxConcurrency: 4,
-    costCeiling: 10,
   },
-  costCeiling: 3,
+  costCeiling: 23,
   confidence: 0.95,
 })
 ```
 
+`costCeiling` is one limit shared by optimizer-model calls, train and selection evaluations, and final test scoring.
 `comparison.scores` contains the final-case baseline score, selected score, lift, simultaneous interval, cost status, duration, and selected surface for each method.
 Official method scores contain optimizer and bridge package versions, source revisions and source-tree hashes, Python runtime, custom engine module hashes, compatible run ID, exact attempt ID, resume status, evaluation count, artifact directory, and available optimizer token usage.
 `comparison.pairwise` compares the highest-ranked method with every other method.
@@ -203,7 +203,7 @@ Install the bridge and the source revision tested by this release:
 
 ```sh
 python -m pip install agent-eval-rpc
-python -m pip install "gepa @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f"
+python -m pip install "gepa[full] @ git+https://github.com/gepa-ai/gepa.git@f919db0a622e2e9f9204779b81fe00cc1b2d808f"
 ```
 
 The published `gepa==0.1.4` wheel does not contain the required Optimize Anything API.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.126.0",
+  "version": "0.126.1",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {