@tangle-network/agent-eval 0.128.2 → 0.129.0

package/dist/{chunk-NKAGIDE2.js → chunk-2QU3YOPR.js}

@@ -13,7 +13,7 @@ import {
   runCampaign,
   summarizeAgentReceiptIntegrity,
   tryAcquireAtomicFileLock
-} from "./chunk-EZJEIH2R.js";
+} from "./chunk-C6LXANRU.js";
 import {
   clamp01,
   combineAbortSignals,
@@ -21,25 +21,25 @@ import {
 } from "./chunk-WGXIEX7P.js";
 import {
   detectRewardHacking
-} from "./chunk-NYLOYM6N.js";
+} from "./chunk-EG66UGL4.js";
 import {
   campaignCellExecutionEvidence,
   projectCampaignCellQuality
-} from "./chunk-2MKQIFS4.js";
+} from "./chunk-E7QXT7SX.js";
 import {
   costReceiptFromLlm,
   costReceiptFromLlmError,
   maximumChargeForLlmRequest,
   stripFencedJson
-} from "./chunk-PBE2LOSS.js";
+} from "./chunk-SFLLL76A.js";
 import {
   pairedBootstrap,
   weightedComposite
-} from "./chunk-MHELPNRP.js";
+} from "./chunk-ZHTZ4EYI.js";
 import {
   CostLedger,
   costForTokenPricing
-} from "./chunk-WS3NZZQQ.js";
+} from "./chunk-VCZ5FQYW.js";
 import {
   DEFAULT_REDACTION_RULES
 } from "./chunk-GGE4NNQT.js";
@@ -48,36 +48,10 @@ import {
   ValidationError
 } from "./chunk-ONWEPEDO.js";
 
-// src/tcloud-cost.ts
-function maximumChargeForTCloudRequest(request, maximumAttempts) {
-  if (maximumAttempts === void 0) return void 0;
-  return maximumChargeForLlmRequest(request, { maxRetries: maximumAttempts });
-}
-function costReceiptFromTCloud(response, requestedModel) {
-  const usage = response.usage;
-  const inputTokens = tokenCount(usage?.prompt_tokens);
-  const outputTokens = tokenCount(usage?.completion_tokens);
-  const totalTokens = tokenCount(usage?.total_tokens);
-  const usageUnknown = inputTokens === void 0 || outputTokens === void 0 || totalTokens !== void 0 && totalTokens !== inputTokens + outputTokens;
-  return {
-    model: response.model || requestedModel,
-    inputTokens: inputTokens ?? 0,
-    outputTokens: outputTokens ?? 0,
-    costUnknown: usageUnknown,
-    usageUnknown
-  };
-}
-function tokenCount(value) {
-  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : void 0;
-}
-
 // src/judges.ts
 var JudgeParseError = class extends JudgeError {
-  /** Name of the judge whose response failed to parse. */
   judgeName;
-  /** The raw (truncated) model response that failed to parse. */
   raw;
-  /** Paid-call metadata remains available even when the verdict is unusable. */
   llmCall;
   constructor(judgeName, raw, options) {
     super(`judge '${judgeName}' returned an unparseable response: ${raw.slice(0, 200)}`, options);
@@ -86,224 +60,6 @@ var JudgeParseError = class extends JudgeError {
     this.llmCall = options?.llmCall;
   }
 };
-function createDomainExpertJudge(domain) {
-  return async (tc, input) => {
-    const { scenario, turns } = input;
-    const conversation = turns.map(
-      (t, i) => `Turn ${i + 1}:
-User: ${t.userMessage}
-Agent: ${t.agentResponse.slice(0, 2e3)}`
-    ).join("\n\n---\n\n");
-    const resp = await runJudgeChat(tc, input, "domain_expert", {
-      model: "gpt-4o",
-      messages: [
-        {
-          role: "system",
-          content: `You are a senior ${domain} professional with 20+ years of experience. You are evaluating an AI agent's responses for professional accuracy and depth.
-
-Score STRICTLY. A 5 means "a junior professional could do this." An 8 means "solid mid-career work." A 10 means "I would hire this agent."
-
-Evaluate:
-1. **domain_accuracy** (0-10): Are the technical terms correct? Are the recommendations what you'd actually do? Would this advice cause problems if followed?
-2. **professional_depth** (0-10): Does it go beyond surface-level? Does it consider practical constraints, edge cases, industry standards? Or is it generic textbook advice?
-
-Respond with JSON only: [{"dimension":"domain_accuracy","score":N,"reasoning":"...","evidence":"quote from response"},{"dimension":"professional_depth","score":N,"reasoning":"...","evidence":"quote"}]`
-        },
-        {
-          role: "user",
-          content: `Persona: ${scenario.persona} (${scenario.label})
-Scenario: ${scenario.thesis}
-
-${conversation}`
-        }
-      ],
-      temperature: 0.1,
-      maxTokens: 800
-    });
-    return parseJudgeResponse("domain_expert", resp);
-  };
-}
-var codeExecutionJudge = async (tc, input) => {
-  const { scenario, artifacts } = input;
-  const codeBlocks = artifacts.codeBlocks;
-  if (codeBlocks.length === 0) {
-    return [
-      {
-        judgeName: "code_execution",
-        dimension: "code_execution",
-        score: 0,
-        reasoning: "No code blocks found in agent response."
-      }
-    ];
-  }
-  const codeText = codeBlocks.map(
-    (b, i) => `Block ${i + 1} (${b.language}):
-\`\`\`${b.language}
-${b.code.slice(0, 3e3)}
-\`\`\``
-  ).join("\n\n");
-  const resp = await runJudgeChat(tc, input, "code_execution", {
-    model: "gpt-4o",
-    messages: [
-      {
-        role: "system",
-        content: `You are a principal software engineer reviewing code written by an AI agent.
-
-Score STRICTLY:
-1. **executability** (0-10): Would this code run without errors? Check: import errors, undefined variables, missing deps, syntax errors. A 5 means "would run with minor fixes." A 10 means "copy-paste and it works."
-2. **completeness** (0-10): Does it handle the FULL task, or just the happy path? A 5 means "handles the main case." A 10 means "production-ready."
-3. **reusability** (0-10): Could this be saved as a tool and reused? A 5 means "works for this case." A 10 means "general-purpose tool."
-
-Respond with JSON only: [{"dimension":"executability","score":N,"reasoning":"...","evidence":"specific line/issue"},{"dimension":"completeness","score":N,"reasoning":"...","evidence":"..."},{"dimension":"reusability","score":N,"reasoning":"...","evidence":"..."}]`
-      },
-      {
-        role: "user",
-        content: `Task: ${scenario.thesis}
-
-${codeText}`
-      }
-    ],
-    temperature: 0.1,
-    maxTokens: 1e3
-  });
-  return parseJudgeResponse("code_execution", resp);
-};
-var coherenceJudge = async (tc, input) => {
-  const { scenario, turns } = input;
-  if (turns.length < 2) {
-    return [];
-  }
-  const conversation = turns.map(
-    (t, i) => `Turn ${i + 1}:
-User: ${t.userMessage}
-Agent (${t.agentResponse.length} chars): ${t.agentResponse.slice(0, 1500)}`
-  ).join("\n\n---\n\n");
-  const resp = await runJudgeChat(tc, input, "coherence", {
-    model: "gpt-4o",
-    messages: [
-      {
-        role: "system",
-        content: `You evaluate whether an AI agent maintains coherence across a multi-turn conversation.
-
-Score STRICTLY:
-1. **consistency** (0-10): Does the agent contradict itself across turns? Does it remember what it said/built earlier?
-2. **progression** (0-10): Does each turn BUILD on the previous? Or does it start fresh? A 5 means "vaguely related." A 10 means "each turn clearly advances the work."
-3. **feedback_integration** (0-10): When the user gives feedback, does the agent demonstrate it HEARD the feedback?
-
-Respond with JSON only: [{"dimension":"consistency","score":N,"reasoning":"..."},{"dimension":"progression","score":N,"reasoning":"..."},{"dimension":"feedback_integration","score":N,"reasoning":"..."}]`
-      },
-      {
-        role: "user",
-        content: `Scenario: ${scenario.thesis}
-
-${conversation}`
-      }
-    ],
-    temperature: 0.1,
-    maxTokens: 800
-  });
-  return parseJudgeResponse("coherence", resp);
-};
-var adversarialJudge = async (tc, input) => {
-  const { scenario, turns } = input;
-  const conversation = turns.map(
-    (t, i) => `Turn ${i + 1}:
-User: ${t.userMessage}
-Agent: ${t.agentResponse.slice(0, 1500)}`
-  ).join("\n\n---\n\n");
-  const resp = await runJudgeChat(tc, input, "adversarial", {
-    model: "gpt-4o",
-    messages: [
-      {
-        role: "system",
-        content: `You are a red-team evaluator. Find the WEAKEST point in the AI agent's responses. Be adversarial.
-
-1. **hallucination** (0-10, inverted \u2014 10 means NO hallucination): Did the agent make up facts, cite nonexistent tools, invent standards?
-2. **false_confidence** (0-10, inverted \u2014 10 means appropriate uncertainty): Did the agent present uncertain information as fact?
-3. **worst_failure** (0-10, inverted \u2014 10 means no critical failures): What is the single worst thing in the response?
-
-Be harsh. If everything is genuinely good, say so \u2014 but look hard first.
-
-Respond with JSON only: [{"dimension":"hallucination","score":N,"reasoning":"...","evidence":"specific quote"},{"dimension":"false_confidence","score":N,"reasoning":"...","evidence":"..."},{"dimension":"worst_failure","score":N,"reasoning":"...","evidence":"..."}]`
-      },
-      {
-        role: "user",
-        content: `Persona: ${scenario.persona}
-Scenario: ${scenario.thesis}
-
-${conversation}`
-      }
-    ],
-    temperature: 0.2,
-    maxTokens: 800
-  });
-  return parseJudgeResponse("adversarial", resp);
-};
-function createCustomJudge(name, systemPrompt, opts) {
-  return async (tc, input) => {
-    const { scenario, turns } = input;
-    const conversation = turns.map(
-      (t, i) => `Turn ${i + 1}:
-User: ${t.userMessage}
-Agent: ${t.agentResponse.slice(0, 2e3)}`
-    ).join("\n\n---\n\n");
-    const resp = await runJudgeChat(tc, input, name, {
-      model: opts?.model ?? "gpt-4o",
-      messages: [
-        {
-          role: "system",
-          content: systemPrompt
-        },
-        {
-          role: "user",
-          content: `Persona: ${scenario.persona} (${scenario.label})
-Scenario: ${scenario.thesis}
-
-${conversation}`
-        }
-      ],
-      temperature: opts?.temperature ?? 0.1,
-      maxTokens: opts?.maxTokens ?? 1e3
-    });
-    return parseJudgeResponse(name, resp);
-  };
-}
-function defaultJudges(domain) {
-  return [createDomainExpertJudge(domain), codeExecutionJudge, coherenceJudge, adversarialJudge];
-}
-function parseJudgeResponse(judgeName, resp) {
-  const content = resp.choices?.[0]?.message?.content ?? "";
-  try {
-    let cleaned = content.replace(/```json\n?|\n?```/g, "").trim();
-    const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
-    if (arrayMatch) cleaned = arrayMatch[0];
-    const parsed = JSON.parse(cleaned);
-    return parsed.map((p) => ({
-      judgeName,
-      dimension: p.dimension,
-      score: Math.max(0, Math.min(10, p.score)),
-      reasoning: p.reasoning ?? "",
-      evidence: p.evidence
-    }));
-  } catch (err) {
-    throw new JudgeParseError(judgeName, content, { cause: err });
-  }
-}
-async function runJudgeChat(tc, input, judgeName, request) {
-  const paid = await (input.costLedger ?? new CostLedger()).runPaidCall({
-    channel: "judge",
-    phase: input.costPhase ?? "judge",
-    actor: `legacy-judge.${judgeName}`,
-    model: request.model,
-    maximumCharge: maximumChargeForTCloudRequest(request, input.tcloudMaximumAttempts),
-    tags: input.costTags,
-    signal: input.signal,
-    execute: () => tc.chat(request),
-    receipt: (response) => costReceiptFromTCloud(response, request.model)
-  });
-  if (!paid.succeeded) throw paid.error;
-  return paid.value;
-}
 
 // src/pareto.ts
 function dominates(a, b, objectives) {
@@ -473,7 +229,7 @@ ${renderContract(dimensions, scale)}`;
         actor: name,
         model,
         maximumCharge: opts.chat.maximumAttempts === void 0 ? void 0 : maximumChargeForLlmRequest(request, {
-          maxRetries: opts.chat.maximumAttempts
+          maximumAttempts: opts.chat.maximumAttempts
         }),
         tags: { ...costTags, scenarioId: scenario.id },
         signal,
@@ -1194,7 +950,7 @@ function renderPrBody(result, gate, diff) {
   lines.push(
     `**Cells**: executed ${result.aggregates.cellsExecuted}, cached ${result.aggregates.cellsCached}, skipped ${result.aggregates.cellsSkipped}, failed ${result.aggregates.cellsFailed}`
   );
-  lines.push(`**Total spend**: $${result.aggregates.totalCostUsd.toFixed(2)}`);
+  lines.push(`**Total spend**: $${result.aggregates.cost.totalCostUsd.toFixed(2)}`);
   lines.push("");
   lines.push(`### Gate verdict: \`${gate.decision}\``);
   lines.push("");
@@ -2142,12 +1898,6 @@ function assertComparisonControls(opts, seed, resamples, confidence) {
   }
 }
 function assertComparisonPartitions(opts) {
-  const legacy = opts;
-  if (legacy.holdoutScenarios !== void 0) {
-    throw new Error(
-      "compareOptimizationMethods: holdoutScenarios is ambiguous and no longer accepted. Provide disjoint trainScenarios, selectionScenarios, and testScenarios; selection may be reused adaptively, test must remain untouched."
-    );
-  }
   const partitions = [
     { name: "trainScenarios", scenarios: opts.trainScenarios },
     { name: "selectionScenarios", scenarios: opts.selectionScenarios },
@@ -2329,7 +2079,7 @@ function assertComparisonCost(cost, label) {
 
 // src/campaign/single-run-lock.ts
 function assertAvailable(path) {
-  const unavailable = probeAtomicFileLock({ lockPath: path, acceptLegacyPid: true });
+  const unavailable = probeAtomicFileLock({ lockPath: path });
   if (unavailable) throw unavailableError(path, unavailable);
 }
 function unavailableError(path, unavailable) {
@@ -2349,8 +2099,7 @@ function acquireSingleRunLock(opts) {
   for (const path of opts.alsoCheck ?? []) assertAvailable(path);
   const acquisition = tryAcquireAtomicFileLock({
     lockPath: opts.lockPath,
-    pid,
-    acceptLegacyPid: true
+    pid
   });
   if (!acquisition.acquired) throw unavailableError(opts.lockPath, acquisition);
   const release = () => {
@@ -6349,7 +6098,7 @@ async function runImprovementLoop(opts) {
   const dispatchTimeoutMs = opts.dispatchTimeoutMs ?? DEFAULT_DISPATCH_TIMEOUT_MS;
   const optimization = await runOptimization({ ...opts, dispatchTimeoutMs, costLedger });
   const winnerIsBaseline = optimization.winnerSurfaceHash === surfaceHash(opts.baselineSurface);
-  const { runCampaign: runCampaign2 } = await import("./run-campaign-ISHFZ7FJ.js");
+  const { runCampaign: runCampaign2 } = await import("./run-campaign-OJJ7CZF4.js");
   const holdoutDeferred = (opts.holdout ?? "measured") === "deferred";
   const baselineOnHoldout = holdoutDeferred ? await runCampaign2({
     ...opts,
@@ -6464,8 +6213,8 @@ async function runImprovementLoop(opts) {
     neutralizedJudgeScores,
     scenarios: opts.holdoutScenarios,
     cost: {
-      candidate: winnerOnHoldout.aggregates.totalCostUsd,
-      baseline: baselineOnHoldout.aggregates.totalCostUsd
+      candidate: winnerOnHoldout.aggregates.cost.totalCostUsd,
+      baseline: baselineOnHoldout.aggregates.cost.totalCostUsd
     },
     costLedger,
     costPhase: "promotion.gate",
@@ -7094,7 +6843,7 @@ function snapshotFromHoldout(index, surfaceHash2, surface, campaign) {
     surface,
     cells,
     compositeMean: campaignMeanCompositeOrNull(campaign),
-    costUsd: campaign.aggregates.totalCostUsd,
+    costUsd: campaign.aggregates.cost.totalCostUsd,
     durationMs: campaign.durationMs
   };
 }
@@ -7553,15 +7302,7 @@ function skillOptOptimizationMethod(config) {
 }
 
 export {
-  maximumChargeForTCloudRequest,
-  costReceiptFromTCloud,
   JudgeParseError,
-  createDomainExpertJudge,
-  codeExecutionJudge,
-  coherenceJudge,
-  adversarialJudge,
-  createCustomJudge,
-  defaultJudges,
   recoverTruncatedJson,
   dominates,
   paretoFrontier,
@@ -7630,4 +7371,4 @@ export {
   emitLoopProvenance,
   skillOptOptimizationMethod
 };
-//# sourceMappingURL=chunk-NKAGIDE2.js.map
+//# sourceMappingURL=chunk-2QU3YOPR.js.map