agentv 2.11.4 → 2.13.0

package/dist/{chunk-KWUTY5XR.js → chunk-FSBZM3HT.js}

@@ -148,7 +148,7 @@ var require_dist = __commonJS({
   }
 });
 
-// ../../packages/core/dist/chunk-REN5PS7B.js
+// ../../packages/core/dist/chunk-JHER2LQ5.js
 import { constants } from "node:fs";
 import { access, readFile } from "node:fs/promises";
 import path from "node:path";
@@ -4195,7 +4195,7 @@ var coerce = {
 };
 var NEVER = INVALID;
 
-// ../../packages/core/dist/chunk-REN5PS7B.js
+// ../../packages/core/dist/chunk-JHER2LQ5.js
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
 var TEST_MESSAGE_ROLES = TEST_MESSAGE_ROLE_VALUES;
 var TEST_MESSAGE_ROLE_SET = new Set(TEST_MESSAGE_ROLE_VALUES);
@@ -34331,6 +34331,11 @@ async function loadConfig(evalFilePath, repoRoot) {
         continue;
       }
       const config2 = parsed;
+      const requiredVersion = parsed.required_version;
+      if (requiredVersion !== void 0 && typeof requiredVersion !== "string") {
+        logWarning(`Invalid required_version in ${configPath}, expected string`);
+        continue;
+      }
       const guidelinePatterns = config2.guideline_patterns;
       if (guidelinePatterns !== void 0 && !Array.isArray(guidelinePatterns)) {
         logWarning(`Invalid guideline_patterns in ${configPath}, expected array`);
@@ -34354,6 +34359,7 @@ async function loadConfig(evalFilePath, repoRoot) {
         configPath
       );
       return {
+        required_version: requiredVersion,
         guideline_patterns: guidelinePatterns,
         eval_patterns: evalPatterns,
         execution: executionDefaults
@@ -34497,6 +34503,22 @@ function extractTotalBudgetUsd(suite) {
   );
   return void 0;
 }
+function extractFailOnError(suite) {
+  const execution = suite.execution;
+  if (!execution || typeof execution !== "object" || Array.isArray(execution)) {
+    return void 0;
+  }
+  const executionObj = execution;
+  const raw = executionObj.fail_on_error ?? executionObj.failOnError;
+  if (raw === void 0 || raw === null) {
+    return void 0;
+  }
+  if (typeof raw === "boolean") {
+    return raw;
+  }
+  logWarning(`Invalid execution.fail_on_error: ${raw}. Must be true or false. Ignoring.`);
+  return void 0;
+}
 function parseExecutionDefaults(raw, configPath) {
   if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
     return void 0;
@@ -36653,13 +36675,15 @@ async function loadTestSuite(evalFilePath, repoRoot, options) {
   }
   const { tests, parsed } = await loadTestsFromYaml(evalFilePath, repoRoot, options);
   const metadata = parseMetadata(parsed);
+  const failOnError = extractFailOnError(parsed);
   return {
     tests,
     trials: extractTrialsConfig(parsed),
     targets: extractTargetsFromSuite(parsed),
     cacheConfig: extractCacheConfig(parsed),
     totalBudgetUsd: extractTotalBudgetUsd(parsed),
-    ...metadata !== void 0 && { metadata }
+    ...metadata !== void 0 && { metadata },
+    ...failOnError !== void 0 && { failOnError }
   };
 }
 var loadEvalSuite = loadTestSuite;
@@ -46481,6 +46505,16 @@ async function resolveWorkspaceTemplate(templatePath) {
   }
   return { dir: resolved };
 }
+function interpolateArgs(args, context) {
+  const vars = {
+    workspace_path: context.workspacePath,
+    test_id: context.testId,
+    eval_run_id: context.evalRunId,
+    case_input: context.caseInput ?? "",
+    case_metadata: context.caseMetadata ? JSON.stringify(context.caseMetadata) : ""
+  };
+  return args.map((arg) => arg.replace(/\{\{(\w+)\}\}/g, (match, name16) => vars[name16] ?? match));
+}
 async function executeWorkspaceScript(config2, context, failureMode = "fatal") {
   const stdin = JSON.stringify({
     workspace_path: context.workspacePath,
@@ -46490,8 +46524,9 @@ async function executeWorkspaceScript(config2, context, failureMode = "fatal") {
     case_metadata: context.caseMetadata ?? null
   });
   const timeoutMs = config2.timeout_ms ?? (failureMode === "fatal" ? 6e4 : 3e4);
-  const cwd = config2.cwd;
-  const commandArray = config2.command ?? config2.script ?? [];
+  const cwd = config2.cwd ?? context.evalDir;
+  const rawCommand = config2.command ?? config2.script ?? [];
+  const commandArray = interpolateArgs(rawCommand, context);
   const result = await execFileWithStdin(commandArray, stdin, {
     timeoutMs,
     cwd
@@ -46506,6 +46541,10 @@ async function executeWorkspaceScript(config2, context, failureMode = "fatal") {
   }
   return result.stdout;
 }
+var QUALITY_PASS_THRESHOLD = 0.8;
+function classifyQualityStatus(score) {
+  return score >= QUALITY_PASS_THRESHOLD ? "ok" : "quality_failure";
+}
 function usesFileReferencePrompt(provider) {
   return isAgentProvider(provider) || provider.kind === "cli";
 }
@@ -46538,7 +46577,8 @@ async function runEvaluation(options) {
     cleanupWorkspaces,
     trials,
     streamCallbacks,
-    totalBudgetUsd
+    totalBudgetUsd,
+    failOnError
   } = options;
   let useCache = options.useCache;
   if (trials && trials.count > 1 && useCache) {
@@ -46613,6 +46653,7 @@ async function runEvaluation(options) {
   const evaluatorRegistry = buildEvaluatorRegistry(evaluators, resolveJudgeProvider);
   const typeRegistry = createBuiltinRegistry();
   const discoveryBaseDir = evalFilePath ? path37.dirname(path37.resolve(evalFilePath)) : process.cwd();
+  const evalDir = discoveryBaseDir;
   await discoverAssertions(typeRegistry, discoveryBaseDir);
   const providerRegistry = createBuiltinProviderRegistry();
   await discoverProviders(providerRegistry, discoveryBaseDir);
@@ -46708,7 +46749,8 @@ async function runEvaluation(options) {
     const scriptContext = {
       workspacePath: sharedWorkspacePath,
       testId: "__before_all__",
-      evalRunId
+      evalRunId,
+      evalDir
     };
     try {
       beforeAllOutput = await executeWorkspaceScript(suiteWorkspace.before_all, scriptContext);
@@ -46732,6 +46774,7 @@ async function runEvaluation(options) {
   let beforeAllOutputAttached = false;
   let cumulativeBudgetCost = 0;
   let budgetExhausted = false;
+  let failOnErrorTriggered = false;
   const promises = filteredEvalCases.map(
     (evalCase) => limit(async () => {
       const workerId = nextWorkerId++;
@@ -46747,7 +46790,14 @@ async function runEvaluation(options) {
           answer: "",
           target: target.name,
           error: `Suite budget exceeded ($${cumulativeBudgetCost.toFixed(4)} / $${totalBudgetUsd.toFixed(4)})`,
-          budgetExceeded: true
+          budgetExceeded: true,
+          executionStatus: "execution_error",
+          failureStage: "setup",
+          failureReasonCode: "budget_exceeded",
+          executionError: {
+            message: `Suite budget exceeded ($${cumulativeBudgetCost.toFixed(4)} / $${totalBudgetUsd.toFixed(4)})`,
+            stage: "setup"
+          }
         };
         if (onProgress) {
           await onProgress({
@@ -46763,6 +46813,37 @@ async function runEvaluation(options) {
         }
         return budgetResult;
       }
+      if (failOnError === true && failOnErrorTriggered) {
+        const errorMsg = "Halted: execution error encountered with fail_on_error enabled";
+        const haltResult = {
+          timestamp: (now ?? (() => /* @__PURE__ */ new Date()))().toISOString(),
+          testId: evalCase.id,
+          dataset: evalCase.dataset,
+          score: 0,
+          hits: [],
+          misses: [],
+          answer: "",
+          target: target.name,
+          error: errorMsg,
+          executionStatus: "execution_error",
+          failureStage: "setup",
+          failureReasonCode: "error_threshold_exceeded",
+          executionError: { message: errorMsg, stage: "setup" }
+        };
+        if (onProgress) {
+          await onProgress({
+            workerId,
+            testId: evalCase.id,
+            status: "failed",
+            completedAt: Date.now(),
+            error: haltResult.error
+          });
+        }
+        if (onResult) {
+          await onResult(haltResult);
+        }
+        return haltResult;
+      }
       if (onProgress) {
         await onProgress({
           workerId,
@@ -46794,7 +46875,8 @@ async function runEvaluation(options) {
           suiteWorkspaceFile,
           streamCallbacks,
           typeRegistry,
-          repoManager
+          repoManager,
+          evalDir
         };
         let result = trials && trials.count > 1 ? await runEvalCaseWithTrials(runCaseOptions, trials) : await runEvalCase(runCaseOptions);
         if (totalBudgetUsd !== void 0) {
@@ -46814,6 +46896,9 @@ async function runEvaluation(options) {
             }
           }
         }
+        if (failOnError === true && result.executionStatus === "execution_error") {
+          failOnErrorTriggered = true;
+        }
         if (beforeAllOutput && !beforeAllOutputAttached) {
           result = { ...result, beforeAllOutput };
           beforeAllOutputAttached = true;
@@ -46863,7 +46948,9 @@ async function runEvaluation(options) {
         (now ?? (() => /* @__PURE__ */ new Date()))(),
         outcome.reason,
         promptInputs,
-        primaryProvider
+        primaryProvider,
+        "agent",
+        "provider_error"
       );
       results.push(errorResult);
       if (onResult) {
@@ -46875,7 +46962,8 @@ async function runEvaluation(options) {
     const scriptContext = {
       workspacePath: sharedWorkspacePath,
       testId: "__after_all__",
-      evalRunId
+      evalRunId,
+      evalDir
     };
     try {
       const afterAllOutput = await executeWorkspaceScript(
@@ -47005,7 +47093,14 @@ async function runBatchEvaluation(options) {
         availableTargets
       });
       if (providerError) {
-        result = { ...result, error: providerError };
+        result = {
+          ...result,
+          error: providerError,
+          executionStatus: "execution_error",
+          failureStage: "agent",
+          failureReasonCode: "provider_error",
+          executionError: { message: providerError, stage: "agent" }
+        };
       }
     } catch (error40) {
       const errorResult = buildErrorResult(
@@ -47014,7 +47109,9 @@ async function runBatchEvaluation(options) {
         nowFn(),
         error40,
         promptInputs,
-        provider
+        provider,
+        "evaluator",
+        "evaluator_error"
       );
       results.push(errorResult);
       if (onResult) {
@@ -47070,7 +47167,8 @@ async function runEvalCase(options) {
     sharedBaselineCommit,
     suiteWorkspaceFile,
     typeRegistry: providedTypeRegistry,
-    repoManager
+    repoManager,
+    evalDir
   } = options;
   const formattingMode = usesFileReferencePrompt(provider) ? "agent" : "lm";
   const promptInputs = await buildPromptInputs(evalCase, formattingMode);
@@ -47103,7 +47201,9 @@ async function runEvalCase(options) {
           nowFn(),
           new Error(`Failed to create workspace: ${message}`),
           promptInputs,
-          provider
+          provider,
+          "setup",
+          "template_error"
         );
       }
     }
@@ -47123,7 +47223,9 @@ async function runEvalCase(options) {
           nowFn(),
           new Error(`Failed to materialize repos: ${message}`),
           promptInputs,
-          provider
+          provider,
+          "repo_setup",
+          "clone_error"
         );
       }
     }
@@ -47133,7 +47235,8 @@ async function runEvalCase(options) {
         testId: evalCase.id,
         evalRunId: evalRunId ?? "",
         caseInput: evalCase.question,
-        caseMetadata: evalCase.metadata
+        caseMetadata: evalCase.metadata,
+        evalDir
       };
       try {
         beforeAllOutput = await executeWorkspaceScript(
@@ -47152,7 +47255,9 @@ async function runEvalCase(options) {
           nowFn(),
           new Error(`before_all script failed: ${message}`),
           promptInputs,
-          provider
+          provider,
+          "setup",
+          "script_error"
         );
       }
     }
@@ -47163,7 +47268,8 @@ async function runEvalCase(options) {
       testId: evalCase.id,
       evalRunId: evalRunId ?? "",
       caseInput: evalCase.question,
-      caseMetadata: evalCase.metadata
+      caseMetadata: evalCase.metadata,
+      evalDir
     };
     try {
       beforeEachOutput = await executeWorkspaceScript(
@@ -47178,7 +47284,9 @@ async function runEvalCase(options) {
         nowFn(),
         new Error(`before_each script failed: ${message}`),
         promptInputs,
-        provider
+        provider,
+        "setup",
+        "script_error"
       );
     }
   }
@@ -47219,7 +47327,9 @@ async function runEvalCase(options) {
         nowFn(),
         error40,
         promptInputs,
-        provider
+        provider,
+        "agent",
+        "provider_error"
       );
       if (workspacePath) {
         if (forceCleanup) {
@@ -47238,7 +47348,9 @@ async function runEvalCase(options) {
       nowFn(),
       lastError ?? new Error("Provider did not return a response"),
       promptInputs,
-      provider
+      provider,
+      "agent",
+      "provider_error"
     );
     if (workspacePath) {
       if (forceCleanup) {
@@ -47294,7 +47406,8 @@ async function runEvalCase(options) {
       testId: evalCase.id,
       evalRunId: evalRunId ?? "",
       caseInput: evalCase.question,
-      caseMetadata: evalCase.metadata
+      caseMetadata: evalCase.metadata,
+      evalDir
     };
     try {
       afterEachOutput = await executeWorkspaceScript(
@@ -47330,7 +47443,18 @@ async function runEvalCase(options) {
       fileChanges,
       workspacePath
     });
-    const finalResult = providerError ? { ...result, error: providerError, beforeAllOutput, beforeEachOutput, afterEachOutput } : { ...result, beforeAllOutput, beforeEachOutput, afterEachOutput };
+    const executionStatus = providerError ? "execution_error" : classifyQualityStatus(result.score);
+    const finalResult = providerError ? {
+      ...result,
+      error: providerError,
+      executionStatus,
+      failureStage: "agent",
+      failureReasonCode: "provider_error",
+      executionError: { message: providerError, stage: "agent" },
+      beforeAllOutput,
+      beforeEachOutput,
+      afterEachOutput
+    } : { ...result, executionStatus, beforeAllOutput, beforeEachOutput, afterEachOutput };
     const isFailure = !!finalResult.error || finalResult.score < 0.5;
     if (workspacePath && !isSharedWorkspace) {
       if (forceCleanup) {
@@ -47351,7 +47475,9 @@ async function runEvalCase(options) {
       nowFn(),
       error40,
       promptInputs,
-      provider
+      provider,
+      "evaluator",
+      "evaluator_error"
     );
     if (workspacePath && !isSharedWorkspace) {
       if (forceCleanup) {
@@ -47389,7 +47515,10 @@ async function runEvalCaseWithTrials(options, trialsConfig) {
       verdict: trialVerdict,
       scores: result.scores,
       error: result.error,
-      costUsd: trialCost
+      costUsd: trialCost,
+      executionStatus: result.executionStatus,
+      failureStage: result.failureStage,
+      failureReasonCode: result.failureReasonCode
     };
     trialResults.push(trial);
     if (trialCost !== void 0) {
@@ -47414,12 +47543,22 @@ async function runEvalCaseWithTrials(options, trialsConfig) {
     0
   );
   const baseResult = allResults[bestTrialIndex];
+  const hasOk = trialResults.some((t) => t.executionStatus === "ok");
+  const allExecutionError = trialResults.length > 0 && trialResults.every((t) => t.executionStatus === "execution_error");
+  const aggregateExecutionStatus = hasOk ? "ok" : allExecutionError ? "execution_error" : "quality_failure";
+  const aggregateFailureStage = aggregateExecutionStatus === "ok" ? void 0 : baseResult.failureStage;
+  const aggregateFailureReasonCode = aggregateExecutionStatus === "ok" ? void 0 : baseResult.failureReasonCode;
+  const aggregateExecutionError = aggregateExecutionStatus === "execution_error" ? baseResult.executionError : void 0;
   return {
     ...baseResult,
     score,
     trials: trialResults,
     aggregation,
-    costLimited: costLimited || void 0
+    costLimited: costLimited || void 0,
+    executionStatus: aggregateExecutionStatus,
+    failureStage: aggregateFailureStage,
+    failureReasonCode: aggregateFailureReasonCode,
+    executionError: aggregateExecutionError
   };
 }
 async function evaluateCandidate(options) {
@@ -47520,7 +47659,8 @@ async function evaluateCandidate(options) {
     scores,
     trace: trace2,
     output,
-    fileChanges
+    fileChanges,
+    executionStatus: classifyQualityStatus(score.score)
   };
 }
 async function runEvaluatorsForCase(options) {
@@ -47825,7 +47965,7 @@ async function invokeProvider(provider, options) {
     }
   }
 }
-function buildErrorResult(evalCase, targetName, timestamp, error40, promptInputs, provider) {
+function buildErrorResult(evalCase, targetName, timestamp, error40, promptInputs, provider, failureStage, failureReasonCode) {
   const message = error40 instanceof Error ? error40.message : String(error40);
   let agentRequest;
   let lmRequest;
@@ -47868,7 +48008,11 @@ function buildErrorResult(evalCase, targetName, timestamp, error40, promptInputs
     target: targetName,
     requests,
     input,
-    error: message
+    error: message,
+    executionStatus: "execution_error",
+    failureStage,
+    failureReasonCode,
+    executionError: { message, stage: failureStage }
   };
 }
 function extractProviderError(response) {
@@ -48767,6 +48911,7 @@ export {
   extractTargetsFromTestCase,
   extractTrialsConfig,
   extractCacheConfig,
+  extractFailOnError,
   detectFormat,
   buildPromptInputs,
   readTestSuiteMetadata,
@@ -48866,4 +49011,4 @@ export {
   OtelStreamingObserver,
   createAgentKernel
 };
-//# sourceMappingURL=chunk-KWUTY5XR.js.map
+//# sourceMappingURL=chunk-FSBZM3HT.js.map