@tangle-network/agent-eval 0.125.0 → 0.126.0

package/dist/contract/index.js

@@ -14,17 +14,19 @@ import {
 import {
   analyzeRuns,
   summarizeExecution
-} from "../chunk-SJT4OBVL.js";
+} from "../chunk-SDPM6554.js";
 import {
   REFERENCE_EQUIVALENCE_INPUT_LIMITS,
   REFERENCE_EQUIVALENCE_JUDGE_VERSION,
+  assertOptimizationResult,
   buildEvidenceVector,
+  compareOptimizationMethods,
   composeGate,
   createReferenceEquivalenceJudge,
   defaultProductionGate,
   emitLoopProvenance,
-  evolutionaryProposer,
-  gepaProposer,
+  externalTextOptimizationMethod,
+  gepaOptimizationMethod,
   heldOutGate,
   heldoutSignificance,
   llmJudge,
@@ -35,9 +37,10 @@ import {
   runEval,
   runImprovementLoop,
   runReferenceEquivalenceJudge,
+  skillOptOptimizationMethod,
   surfaceContentHash,
   surfaceHash
-} from "../chunk-PMITBABE.js";
+} from "../chunk-NTOV7RU5.js";
 import {
   campaignSplitDigest,
   createRunCostLedger,
@@ -45,24 +48,24 @@ import {
   inMemoryCampaignStorage,
   resolveRunDir,
   runCampaign
-} from "../chunk-D5JZ7UDZ.js";
+} from "../chunk-UCLVDLCH.js";
 import {
   buildDefaultAnalystRegistry,
   createChatClient
-} from "../chunk-A6GT67HT.js";
+} from "../chunk-CM4OILD2.js";
 import "../chunk-HHWE3POT.js";
-import "../chunk-DT7OXY3C.js";
+import "../chunk-UI4YMIN2.js";
 import {
   FileSystemOutcomeStore,
   InMemoryOutcomeStore
 } from "../chunk-3RF76KTD.js";
 import "../chunk-ARU2PZFM.js";
-import "../chunk-EQUK3RFS.js";
+import "../chunk-J5SQWP6Y.js";
 import "../chunk-DPZAEKA6.js";
 import {
   pairedBootstrap
 } from "../chunk-PJQFMIOX.js";
-import "../chunk-GRCDRKII.js";
+import "../chunk-WS3NZZQQ.js";
 import "../chunk-VI2UW6B6.js";
 import {
   recordAggregateMeasurements,
@@ -75,9 +78,8 @@ import "../chunk-IR3KBHOY.js";
 import "../chunk-PC4UYEBM.js";
 import {
   parseRunRecordSafe
-} from "../chunk-LOW3U7JZ.js";
+} from "../chunk-IILEIWGW.js";
 import "../chunk-MA6HLL3S.js";
-import "../chunk-GC4ATIKK.js";
 import "../chunk-VSMTAMNK.js";
 import "../chunk-ONWEPEDO.js";
 import {
@@ -98,16 +100,95 @@ var SelfImproveRunError = class extends Error {
     this.receipts = ledger.list();
   }
 };
-function splitTrainHoldout(scenarios, fraction) {
-  function hash(s) {
-    let h = 2166136261 >>> 0;
-    for (let i = 0; i < s.length; i++) {
-      h ^= s.charCodeAt(i);
-      h = Math.imul(h, 16777619) >>> 0;
+function assertSelfImproveSearchMode(opts) {
+  if (opts.method && opts.proposer) {
+    throw new Error("selfImprove: method and proposer are mutually exclusive");
+  }
+  if (!opts.method) {
+    if (opts.selectionScenarios !== void 0) {
+      throw new Error("selfImprove: selectionScenarios requires method");
+    }
+    return;
+  }
+  if (typeof opts.method.name !== "string" || !opts.method.name.trim() || opts.method.name.trim() !== opts.method.name || typeof opts.method.optimize !== "function") {
+    throw new Error("selfImprove: method must have a trimmed name and optimize(input)");
+  }
+  const budget = opts.budget;
+  if (budget?.generations !== void 0 && budget.generations !== 1) {
+    throw new Error("selfImprove: method owns its rounds; budget.generations must be 1 when set");
+  }
+  if (budget?.populationSize !== void 0 && budget.populationSize !== 1) {
+    throw new Error(
+      "selfImprove: method owns its candidates; budget.populationSize must be 1 when set"
+    );
+  }
+  if (budget?.candidateConcurrency !== void 0 || budget?.maxImprovementShots !== void 0 || opts.analyzeGeneration !== void 0 || opts.findings !== void 0) {
+    throw new Error(
+      "selfImprove: candidateConcurrency, maxImprovementShots, analyzeGeneration, and findings apply only to proposer mode"
+    );
+  }
+}
+function splitMethodPartitions(searchScenarios, explicitSelection, fraction) {
+  if (!Number.isFinite(fraction) || fraction <= 0 || fraction >= 1) {
+    throw new Error("selfImprove: budget.selectionFraction must be in (0, 1)");
+  }
+  const byId = /* @__PURE__ */ new Map();
+  for (const scenario of searchScenarios) {
+    if (byId.has(scenario.id)) {
+      throw new Error(`selfImprove: duplicate scenario id '${scenario.id}'`);
     }
-    return h;
+    byId.set(scenario.id, scenario);
   }
-  const sorted = [...scenarios].sort((a, b) => hash(a.id) - hash(b.id));
+  if (explicitSelection) {
+    if (explicitSelection.length === 0) {
+      throw new Error("selfImprove: selectionScenarios must not be empty");
+    }
+    const selectionIds = /* @__PURE__ */ new Set();
+    for (const scenario of explicitSelection) {
+      if (!byId.has(scenario.id)) {
+        throw new Error(
+          `selfImprove: selection scenario '${scenario.id}' is absent from the non-final cases`
+        );
+      }
+      if (selectionIds.has(scenario.id)) {
+        throw new Error(`selfImprove: duplicate selection scenario id '${scenario.id}'`);
+      }
+      selectionIds.add(scenario.id);
+    }
+    const train = searchScenarios.filter((scenario) => !selectionIds.has(scenario.id));
+    if (train.length === 0) {
+      throw new Error("selfImprove: method train split is empty");
+    }
+    return {
+      train,
+      selection: explicitSelection.map((scenario) => byId.get(scenario.id))
+    };
+  }
+  if (searchScenarios.length < 2) {
+    throw new Error("selfImprove: method requires at least two non-final scenarios");
+  }
+  const sorted = [...searchScenarios].sort(
+    (a, b) => stableScenarioHash(a.id) - stableScenarioHash(b.id)
+  );
+  const count = Math.max(1, Math.min(sorted.length - 1, Math.round(sorted.length * fraction)));
+  return {
+    selection: sorted.slice(0, count),
+    train: sorted.slice(count)
+  };
+}
+function safeRunComponent(value) {
+  return value.replace(/[^a-zA-Z0-9._-]/g, "_");
+}
+function stableScenarioHash(value) {
+  let hash = 2166136261 >>> 0;
+  for (let index = 0; index < value.length; index++) {
+    hash ^= value.charCodeAt(index);
+    hash = Math.imul(hash, 16777619) >>> 0;
+  }
+  return hash;
+}
+function splitTrainHoldout(scenarios, fraction) {
+  const sorted = [...scenarios].sort((a, b) => stableScenarioHash(a.id) - stableScenarioHash(b.id));
   const nHoldout = Math.max(1, Math.min(sorted.length - 1, Math.round(sorted.length * fraction)));
   return {
     holdout: sorted.slice(0, nHoldout),
@@ -137,7 +218,7 @@ function winnerSearchCampaign(result) {
 }
 async function selfImprove(opts) {
   const startedAt = Date.now();
-  const requestedRunDir = opts.runDir ?? `mem://selfImprove-${startedAt}`;
+  const requestedRunDir = opts.runDir ?? (opts.method ? `.agent-eval/runs/self-improve-${startedAt}` : `mem://selfImprove-${startedAt}`);
   const runDir = resolveRunDir(requestedRunDir);
   const storage = opts.storage ?? (runDir.startsWith("mem://") ? inMemoryCampaignStorage() : fsCampaignStorage());
   const costLedger = createRunCostLedger({
@@ -153,8 +234,9 @@ async function selfImprove(opts) {
 }
 async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
   const budget = opts.budget ?? {};
-  const generations = budget.generations ?? 3;
-  const populationSize = budget.populationSize ?? 2;
+  assertSelfImproveSearchMode(opts);
+  const generations = opts.method ? 1 : budget.generations ?? 3;
+  const populationSize = opts.method ? 1 : budget.populationSize ?? 2;
   const maxConcurrency = budget.maxConcurrency ?? 2;
   const holdoutFraction = budget.holdoutFraction ?? 0.25;
   const holdoutMode = budget.holdout ?? "measured";
@@ -173,17 +255,55 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
   if (holdout.length === 0 && !holdoutDeferred) {
     throw new Error("selfImprove: holdout split is empty. Pass more scenarios.");
   }
-  const proposer = opts.proposer ?? gepaProposer({
-    llm: {
-      baseUrl: opts.llm?.baseUrl ?? "https://router.tangle.tools/v1",
-      apiKey: opts.llm?.apiKey ?? process.env.OPENAI_API_KEY ?? ""
-    },
-    model: opts.llm?.model ?? "anthropic/claude-sonnet-4.6",
-    target: opts.proposerTarget ?? "agent surface (system prompt or config) being optimized by selfImprove",
-    // Pass-through: when unset, gepaProposer falls back to its own
-    // domain-neutral engineering primitives.
-    mutationPrimitives: opts.mutationPrimitives
-  });
+  if (generations > 0 && !opts.proposer && !opts.method) {
+    throw new Error(
+      "selfImprove: method or proposer is required when budget.generations is greater than zero"
+    );
+  }
+  let optimizationResult;
+  const methodPartitions = opts.method ? splitMethodPartitions(train, opts.selectionScenarios, budget.selectionFraction ?? 0.25) : void 0;
+  const proposer = opts.method ? {
+    kind: `method:${opts.method.name}`,
+    propose: async (context) => {
+      if (context.generation > 0) return [];
+      const result2 = await opts.method.optimize(
+        Object.freeze({
+          baselineSurface: structuredClone(context.currentSurface),
+          trainScenarios: Object.freeze(
+            methodPartitions.train.map((scenario) => structuredClone(scenario))
+          ),
+          selectionScenarios: Object.freeze(
+            methodPartitions.selection.map((scenario) => structuredClone(scenario))
+          ),
+          dispatchWithSurface: opts.agent,
+          judges: Object.freeze([opts.judge]),
+          runDir: `${runDir}/optimization/${safeRunComponent(opts.method.name)}`,
+          seed: 42,
+          runOptions: Object.freeze({
+            storage,
+            maxConcurrency,
+            reps: budget.reps,
+            dispatchTimeoutMs: opts.dispatchTimeoutMs,
+            expectUsage,
+            costCeiling: budget.dollars
+          }),
+          costLedger
+        })
+      );
+      assertOptimizationResult(opts.method.name, result2);
+      optimizationResult = structuredClone(result2);
+      return [
+        {
+          surface: structuredClone(result2.winnerSurface),
+          label: opts.method.name,
+          rationale: `${opts.method.name} selected this surface without final cases.`
+        }
+      ];
+    }
+  } : opts.proposer ?? {
+    kind: "baseline-only",
+    propose: async () => []
+  };
   const gate = opts.gate ?? defaultProductionGate({
     holdoutScenarios: holdout,
     deltaThreshold: 0.05
@@ -201,7 +321,6 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
     populationSize,
     maxGenerations: generations,
     candidateConcurrency: budget.candidateConcurrency,
-    promoteTopK: budget.promoteTopK,
     reps: budget.reps,
     maxImprovementShots: budget.maxImprovementShots,
     holdoutScenarios: holdout,
@@ -293,6 +412,14 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
       totalCostUsd: totalCost,
       totalDurationMs: durationMs
     }),
+    ...optimizationResult ? {
+      optimizationMethod: {
+        name: opts.method.name,
+        cost: structuredClone(optimizationResult.cost),
+        ...optimizationResult.durationMs === void 0 ? {} : { durationMs: optimizationResult.durationMs },
+        ...optimizationResult.provenance === void 0 ? {} : { provenance: structuredClone(optimizationResult.provenance) }
+      }
+    } : {},
     storage,
     hostedClient: opts.hostedTenant ? createHostedClient(opts.hostedTenant) : void 0
   });
@@ -314,6 +441,14 @@ async function runSelfImprove(opts, costLedger, startedAt, runDir, storage) {
     totalCostUsd: totalCost,
     cost,
     receipts: costLedger.list(),
+    ...optimizationResult ? {
+      optimization: {
+        name: opts.method.name,
+        cost: structuredClone(optimizationResult.cost),
+        ...optimizationResult.durationMs === void 0 ? {} : { durationMs: optimizationResult.durationMs },
+        ...optimizationResult.provenance === void 0 ? {} : { provenance: structuredClone(optimizationResult.provenance) }
+      }
+    } : {},
     insight,
     ...power ? { power } : {},
     raw: result
@@ -487,17 +622,14 @@ function defineAgentEval(defaults) {
       const {
         budget: budgetOverride,
         hostedTenant: hostedTenantOverride,
-        llm: llmOverride,
         ...topLevelOverrides
       } = opts;
       const merged = mergeDefined(defaults, topLevelOverrides);
       const budget = mergeBudget(defaults.budget, budgetOverride);
-      const llm = mergeLlm(defaults.llm, llmOverride);
       const hostedTenant = mergeHostedTenant(defaults.hostedTenant, hostedTenantOverride);
       return selfImprove({
         ...merged,
         ...budget ? { budget } : {},
-        ...llm ? { llm } : {},
         ...hostedTenant ? { hostedTenant } : {}
       });
     }
@@ -522,9 +654,6 @@ function mergeBudget(defaults, overrides) {
   if (merged?.reps !== void 0) merged.reps = requirePositiveInteger(merged.reps, "budget.reps");
   return merged;
 }
-function mergeLlm(defaults, overrides) {
-  return mergeOptionalObject(defaults, overrides);
-}
 function mergeHostedTenant(defaults, overrides) {
   const merged = mergeOptionalObject(defaults, overrides);
   if (!merged) return void 0;
@@ -1682,6 +1811,7 @@ export {
   buildDefaultAnalystRegistry,
   buildEvidenceVector,
   campaignSplitDigest,
+  compareOptimizationMethods,
   composeGate,
   createChatClient,
   createReferenceEquivalenceJudge,
@@ -1691,7 +1821,7 @@ export {
   diffRunBaselineToWinner,
   diffRuns,
   evalReportingSuite,
-  evolutionaryProposer,
+  externalTextOptimizationMethod,
   fromClaudeCodeSession,
   fromCodexSession,
   fromFeedbackTable,
@@ -1702,7 +1832,7 @@ export {
   fromPigraphSession,
   fromRunRecordDir,
   fsCampaignStorage,
-  gepaProposer,
+  gepaOptimizationMethod,
   heldOutGate,
   inMemoryCampaignStorage,
   llmJudge,
@@ -1722,6 +1852,7 @@ export {
   sealCandidateBenchmarkTask,
   sealCandidateExperiment,
   selfImprove,
+  skillOptOptimizationMethod,
   summarizeExecution,
   verifyCandidateBenchmarkSuite,
   verifyCandidateBenchmarkSuiteInputs,