@tangle-network/agent-eval 0.94.0 → 0.95.0

package/dist/contract/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/contract/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoFG;AAMH,YAAY,EACV,kBAAkB,EAClB,sBAAsB,EACtB,kBAAkB,EAClB,iBAAiB,EACjB,cAAc,EACd,mBAAmB,EACnB,WAAW,EACX,eAAe,EACf,UAAU,IAAI,QAAQ,EACtB,IAAI,EACJ,WAAW,EACX,YAAY,EACZ,UAAU,EACV,mBAAmB,EACnB,gBAAgB,EAChB,WAAW,EACX,cAAc,EACd,UAAU,EACV,cAAc,EACd,OAAO,EACP,oBAAoB,EACpB,eAAe,EACf,QAAQ,EACR,aAAa,EACb,eAAe,GAChB,MAAM,mBAAmB,CAAA;AAI1B,OAAO,EAAE,KAAK,cAAc,EAAE,OAAO,EAAE,MAAM,8BAA8B,CAAA;AAC3E,OAAO,EACL,KAAK,yBAAyB,EAC9B,KAAK,wBAAwB,EAC7B,kBAAkB,GACnB,MAAM,0CAA0C,CAAA;AACjD,OAAO,EAAE,KAAK,kBAAkB,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAA;AAI/E,OAAO,EACL,KAAK,2BAA2B,EAChC,oBAAoB,GACrB,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,KAAK,mBAAmB,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAA;AAInF,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EACL,KAAK,4BAA4B,EACjC,qBAAqB,GACtB,MAAM,2CAA2C,CAAA;AAClD,OAAO,EAAE,KAAK,kBAAkB,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAA;AACrF,OAAO,EACL,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,0BAA0B,EAC/B,mBAAmB,EACnB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,6BAA6B,EAClC,KAAK,kBAAkB,EACvB,KAAK,eAAe,EACpB,YAAY,EACZ,sBAAsB,GACvB,MAAM,oCAAoC,CAAA;AAI3C,OAAO,EACL,KAAK,eAAe,EACpB,iBAAiB,EACjB,uBAAuB,GACxB,MAAM,qBAAqB,CAAA;AAI5B,OAAO,EACL,KAAK,iBAAiB,EACtB,sBAAsB,EACtB,KAAK,6BAA6B,EAClC,oBAAoB,EACpB,KAAK,YAAY,GAClB,MAAM,4BAA4B,CAAA;AAInC,YAAY,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,wBAAwB,EAC7B,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,KAAK,gBAAgB,EACrB,eAAe,GAChB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,cAAc,EACnB,KAAK,kBAAkB,EACvB,KAAK,wBAAwB,EAC7B,KAAK,iBAAiB,EACtB,WAAW,GACZ,MAAM,gBAAgB,CAAA;AAYvB,OAAO,EACL,2BAA2B,EAC3B,KAAK,6BAA6B,GACnC,MAAM,6BAA6B,CAAA;AACpC,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACtD,YAAY,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AACxD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAA;AAC5C,YAAY,EACV,qBAAqB,EACrB,aAAa,EACb,iBAAiB,EACjB,YAAY,EACZ,WAAW,EACX,yBAAyB,EACzB,cAAc,EACd,cAAc,EACd,kBAAkB,GACnB,MAAM,kBAAkB,CAAA;AAIzB,OAAO,EACL,eAAe,EACf,uBAAuB,EACvB,QAAQ,EACR,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,WAAW,GACjB,MAAM,QAAQ,CAAA;AAMf,OAAO,EACL,KAAK,qBAAqB,EAC1B,KAAK,yBAAyB,EAC9B,KAAK,sBAAsB,EAC3B,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,EACxB,KAAK,0BAA0B,EAC/B,KAAK,6BAA6B,EAClC,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,EAC7B,KAAK,uBAAuB,EAC5B,KAAK,oBAAoB,EACzB,qBAAqB,EACrB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACnB,mBAAmB,EACnB,aAAa,EACb,kBAAkB,EAClB,aAAa,EACb,KAAK,oBAAoB,EACzB,KAAK,+BAA+B,EACpC,eAAe,EACf,mBAAmB,EACnB,6BAA6B,GAC9B,MAAM,UAAU,CAAA"}

package/dist/contract/index.js

@@ -14,31 +14,31 @@ import {
   buildEvidenceVector,
   composeGate,
   defaultProductionGate,
-  evolutionaryDriver,
+  evolutionaryProposer,
   paretoPolicy,
   paretoSignificanceGate,
   runEval
-} from "../chunk-SD2YFWQQ.js";
+} from "../chunk-KKWJD5E6.js";
 import {
   analyzeRuns
-} from "../chunk-2KNZHH3P.js";
+} from "../chunk-Z6L6YSU6.js";
 import {
   emitLoopProvenance,
-  gepaDriver,
+  gepaProposer,
   heldOutGate,
   runImprovementLoop,
   surfaceContentHash
-} from "../chunk-EGPMSBEZ.js";
+} from "../chunk-7QTQKIDD.js";
 import {
   fsCampaignStorage,
   inMemoryCampaignStorage,
   runCampaign
-} from "../chunk-2K6UUZ7P.js";
+} from "../chunk-2T4EZACH.js";
 import "../chunk-VI2UW6B6.js";
 import {
   buildDefaultAnalystRegistry
-} from "../chunk-MIFZUPEK.js";
-import "../chunk-Q5LIB7BC.js";
+} from "../chunk-AQ5WQAIV.js";
+import "../chunk-S4SYLDFX.js";
 import "../chunk-5HRORJQY.js";
 import {
   FileSystemOutcomeStore,
@@ -51,7 +51,11 @@ import "../chunk-56GC6VYO.js";
 import "../chunk-GGE4NNQT.js";
 import "../chunk-UFSG7ACU.js";
 import "../chunk-5BKGXME7.js";
-import "../chunk-QMUEXQJS.js";
+import {
+  LLM_INPUT_TOKEN_ATTR_KEYS,
+  LLM_MODEL_ATTR_KEYS,
+  LLM_OUTPUT_TOKEN_ATTR_KEYS
+} from "../chunk-YBIGNSCZ.js";
 import "../chunk-PC4UYEBM.js";
 import "../chunk-VSMTAMNK.js";
 import "../chunk-3BFEG2F6.js";
@@ -115,13 +119,13 @@ async function selfImprove(opts) {
   if (holdout.length === 0) {
     throw new Error("selfImprove: holdout split is empty. Pass more scenarios.");
   }
-  const driver = opts.driver ?? gepaDriver({
+  const proposer = opts.proposer ?? gepaProposer({
     llm: {
       baseUrl: opts.llm?.baseUrl ?? "https://router.tangle.tools/v1",
       apiKey: opts.llm?.apiKey ?? process.env.OPENAI_API_KEY ?? ""
     },
     model: opts.llm?.model ?? "anthropic/claude-sonnet-4.6",
-    target: opts.driverTarget ?? "agent surface (system prompt or config) being optimized by selfImprove",
+    target: opts.proposerTarget ?? "agent surface (system prompt or config) being optimized by selfImprove",
     mutationPrimitives: opts.mutationPrimitives ?? DEFAULT_MUTATION_PRIMITIVES
   });
   const gate = opts.gate ?? defaultProductionGate({
@@ -138,7 +142,7 @@ async function selfImprove(opts) {
     scenarios: train,
     baselineSurface: opts.baselineSurface,
     dispatchWithSurface: opts.agent,
-    driver,
+    proposer,
     judges: [opts.judge],
     populationSize,
     maxGenerations: generations,
@@ -365,6 +369,111 @@ function cellsToRunRecords(cells, candidateId, runId, surface) {
   });
 }
 
+// src/contract/define-agent-eval.ts
+function defineAgentEval(defaults) {
+  const defaultEvaluateOptions = evaluateDefaults(defaults);
+  return {
+    scenarios: defaults.scenarios,
+    baselineSurface: defaults.baselineSurface,
+    async evaluate(opts = {}) {
+      const { agent, judge, judges, runDir, scenarios, surface, ...campaignOpts } = opts;
+      const selectedAgent = agent ?? defaults.agent;
+      const selectedSurface = surface ?? defaults.baselineSurface;
+      const selectedRunDir = runDir ?? defaults.runDir ?? `mem://defineAgentEval-${Date.now()}`;
+      const selectedStorage = campaignOpts.storage ?? defaultEvaluateOptions.storage ?? (selectedRunDir.startsWith("mem://") ? inMemoryCampaignStorage() : void 0);
+      const evalOptions = {
+        ...defaultEvaluateOptions,
+        ...campaignOpts,
+        ...selectedStorage ? { storage: selectedStorage } : {},
+        runDir: selectedRunDir,
+        scenarios: scenarios ?? defaults.scenarios,
+        dispatch: (scenario, ctx) => selectedAgent(selectedSurface, scenario, ctx),
+        judges: evaluateJudges(judges, judge ?? defaults.judge)
+      };
+      if (evalOptions.reps !== void 0)
+        evalOptions.reps = requirePositiveInteger(evalOptions.reps, "reps");
+      return runEval(evalOptions);
+    },
+    async improve(opts = {}) {
+      const {
+        budget: budgetOverride,
+        hostedTenant: hostedTenantOverride,
+        llm: llmOverride,
+        ...topLevelOverrides
+      } = opts;
+      const merged = mergeDefined(defaults, topLevelOverrides);
+      const budget = mergeBudget(defaults.budget, budgetOverride);
+      const llm = mergeLlm(defaults.llm, llmOverride);
+      const hostedTenant = mergeHostedTenant(defaults.hostedTenant, hostedTenantOverride);
+      return selfImprove({
+        ...merged,
+        ...budget ? { budget } : {},
+        ...llm ? { llm } : {},
+        ...hostedTenant ? { hostedTenant } : {}
+      });
+    }
+  };
+}
+function evaluateDefaults(defaults) {
+  const out = {};
+  if (defaults.storage) out.storage = defaults.storage;
+  if (defaults.labeledStore) out.labeledStore = defaults.labeledStore;
+  if (defaults.captureSource) out.captureSource = defaults.captureSource;
+  if (defaults.cellPlacement) out.cellPlacement = defaults.cellPlacement;
+  if (defaults.expectUsage) out.expectUsage = defaults.expectUsage;
+  if (defaults.budget?.dollars !== void 0) out.costCeiling = defaults.budget.dollars;
+  if (defaults.budget?.maxConcurrency !== void 0)
+    out.maxConcurrency = defaults.budget.maxConcurrency;
+  if (defaults.budget?.reps !== void 0)
+    out.reps = requirePositiveInteger(defaults.budget.reps, "budget.reps");
+  return out;
+}
+function mergeBudget(defaults, overrides) {
+  const merged = mergeOptionalObject(defaults, overrides);
+  if (merged?.reps !== void 0) merged.reps = requirePositiveInteger(merged.reps, "budget.reps");
+  return merged;
+}
+function mergeLlm(defaults, overrides) {
+  return mergeOptionalObject(defaults, overrides);
+}
+function mergeHostedTenant(defaults, overrides) {
+  const merged = mergeOptionalObject(defaults, overrides);
+  if (!merged) return void 0;
+  if (!merged.endpoint?.trim() || !merged.apiKey?.trim() || !merged.tenantId?.trim()) {
+    throw new Error(
+      "defineAgentEval.improve: hostedTenant requires endpoint, apiKey, and tenantId after merging defaults and overrides"
+    );
+  }
+  return merged;
+}
+function mergeDefined(defaults, overrides) {
+  if (!overrides) return defaults;
+  const merged = { ...defaults };
+  for (const [key, value] of Object.entries(overrides)) {
+    if (value !== void 0) merged[key] = value;
+  }
+  return merged;
+}
+function mergeOptionalObject(defaults, overrides) {
+  if (!defaults && !overrides) return void 0;
+  return mergeDefined(defaults ?? {}, overrides);
+}
+function evaluateJudges(judges, defaultJudge) {
+  if (judges !== void 0) {
+    if (judges.length === 0) {
+      throw new Error("defineAgentEval.evaluate: judges must not be empty");
+    }
+    return judges;
+  }
+  return [defaultJudge];
+}
+function requirePositiveInteger(value, field) {
+  if (!Number.isInteger(value) || value < 1) {
+    throw new Error(`defineAgentEval: ${field} must be a positive integer`);
+  }
+  return value;
+}
+
 // src/contract/diff.ts
 function keyForCell(cell) {
   return JSON.stringify([cell.scenarioId, cell.rep]);
@@ -612,10 +721,10 @@ function fromFeedbackTable(opts) {
 
 // src/contract/intake/otel-spans.ts
 var SCORE_KEYS = ["tangle.score", "eval.score", "score"];
-var MODEL_KEYS = ["tangle.model", "gen_ai.request.model", "llm.model", "model"];
+var MODEL_KEYS = ["tangle.model", ...LLM_MODEL_ATTR_KEYS, "model"];
 var COST_KEYS = ["tangle.cost.usd", "gen_ai.usage.cost_usd", "cost.usd", "cost"];
-var INPUT_TOKEN_KEYS = ["gen_ai.usage.input_tokens", "tangle.tokens.in", "tokens.in"];
-var OUTPUT_TOKEN_KEYS = ["gen_ai.usage.output_tokens", "tangle.tokens.out", "tokens.out"];
+var INPUT_TOKEN_KEYS = [...LLM_INPUT_TOKEN_ATTR_KEYS, "tangle.tokens.in", "tokens.in"];
+var OUTPUT_TOKEN_KEYS = [...LLM_OUTPUT_TOKEN_ATTR_KEYS, "tangle.tokens.out", "tokens.out"];
 var PROMPT_HASH_KEYS = ["tangle.prompt_hash", "prompt.hash"];
 var CONFIG_HASH_KEYS = ["tangle.config_hash", "config.hash"];
 function fromOtelSpans(opts) {
@@ -720,10 +829,11 @@ export {
   buildEvidenceVector,
   composeGate,
   defaultProductionGate,
+  defineAgentEval,
   diffGenerations,
   diffRunBaselineToWinner,
   diffRuns,
-  evolutionaryDriver,
+  evolutionaryProposer,
   fromClaudeCodeSession,
   fromCodexSession,
   fromFeedbackTable,
@@ -733,7 +843,7 @@ export {
   fromPiSession,
   fromPigraphSession,
   fsCampaignStorage,
-  gepaDriver,
+  gepaProposer,
   heldOutGate,
   inMemoryCampaignStorage,
   paretoPolicy,