@agentv/core 2.11.2 → 2.12.0

package/dist/evaluation/validation/index.js

@@ -6,7 +6,7 @@ import {
   findGitRoot,
   isEvaluatorKind,
   resolveFileReference
-} from "../../chunk-REN5PS7B.js";
+} from "../../chunk-7HPKTRFZ.js";
 
 // src/evaluation/validation/file-type.ts
 import { readFile } from "node:fs/promises";

package/dist/index.cjs

@@ -15512,14 +15512,24 @@ var RepoManager = class {
    * Creates on first access, fetches updates on subsequent calls.
    * Returns the absolute path to the cache directory.
    */
-  async ensureCache(source, depth) {
+  async ensureCache(source, depth, resolve) {
     const key = cacheKey(source);
     const cachePath = import_node_path38.default.join(this.cacheDir, key);
     const lockPath = `${cachePath}.lock`;
+    const cacheExists = (0, import_node_fs11.existsSync)(import_node_path38.default.join(cachePath, "HEAD"));
+    if (resolve === "local") {
+      if (cacheExists) {
+        return cachePath;
+      }
+      const url = getSourceUrl(source);
+      throw new Error(
+        `No cache found for \`${url}\`. Run \`agentv cache add --url ${url} --from <local-path>\` to seed it.`
+      );
+    }
     await (0, import_promises27.mkdir)(this.cacheDir, { recursive: true });
     await acquireLock(lockPath);
     try {
-      if ((0, import_node_fs11.existsSync)(import_node_path38.default.join(cachePath, "HEAD"))) {
+      if (cacheExists) {
         const fetchArgs = ["fetch", "--prune"];
         if (depth) {
           fetchArgs.push("--depth", String(depth));
@@ -15546,7 +15556,11 @@ var RepoManager = class {
    */
   async materialize(repo, workspacePath) {
     const targetDir = import_node_path38.default.join(workspacePath, repo.path);
-    const cachePath = await this.ensureCache(repo.source, repo.clone?.depth);
+    const cachePath = await this.ensureCache(
+      repo.source,
+      repo.clone?.depth,
+      repo.checkout?.resolve
+    );
     const cloneArgs = ["clone"];
     if (repo.clone?.depth) {
       cloneArgs.push("--depth", String(repo.clone.depth));
@@ -15692,6 +15706,16 @@ async function resolveWorkspaceTemplate(templatePath) {
 }
 
 // src/evaluation/workspace/script-executor.ts
+function interpolateArgs(args, context2) {
+  const vars = {
+    workspace_path: context2.workspacePath,
+    test_id: context2.testId,
+    eval_run_id: context2.evalRunId,
+    case_input: context2.caseInput ?? "",
+    case_metadata: context2.caseMetadata ? JSON.stringify(context2.caseMetadata) : ""
+  };
+  return args.map((arg) => arg.replace(/\{\{(\w+)\}\}/g, (match, name) => vars[name] ?? match));
+}
 async function executeWorkspaceScript(config, context2, failureMode = "fatal") {
   const stdin = JSON.stringify({
     workspace_path: context2.workspacePath,
@@ -15701,8 +15725,9 @@ async function executeWorkspaceScript(config, context2, failureMode = "fatal") {
     case_metadata: context2.caseMetadata ?? null
   });
   const timeoutMs = config.timeout_ms ?? (failureMode === "fatal" ? 6e4 : 3e4);
-  const cwd = config.cwd;
-  const commandArray = config.command ?? config.script ?? [];
+  const cwd = config.cwd ?? context2.evalDir;
+  const rawCommand = config.command ?? config.script ?? [];
+  const commandArray = interpolateArgs(rawCommand, context2);
   const result = await execFileWithStdin(commandArray, stdin, {
     timeoutMs,
     cwd
@@ -15719,6 +15744,10 @@ async function executeWorkspaceScript(config, context2, failureMode = "fatal") {
 }
 
 // src/evaluation/orchestrator.ts
+var QUALITY_PASS_THRESHOLD = 0.8;
+function classifyQualityStatus(score) {
+  return score >= QUALITY_PASS_THRESHOLD ? "ok" : "quality_failure";
+}
 function usesFileReferencePrompt(provider) {
   return isAgentProvider(provider) || provider.kind === "cli";
 }
@@ -15826,6 +15855,7 @@ async function runEvaluation(options) {
   const evaluatorRegistry = buildEvaluatorRegistry(evaluators, resolveJudgeProvider);
   const typeRegistry = createBuiltinRegistry();
   const discoveryBaseDir = evalFilePath ? import_node_path40.default.dirname(import_node_path40.default.resolve(evalFilePath)) : process.cwd();
+  const evalDir = discoveryBaseDir;
   await discoverAssertions(typeRegistry, discoveryBaseDir);
   const providerRegistry = createBuiltinProviderRegistry();
   await discoverProviders(providerRegistry, discoveryBaseDir);
@@ -15921,7 +15951,8 @@ async function runEvaluation(options) {
     const scriptContext = {
       workspacePath: sharedWorkspacePath,
       testId: "__before_all__",
-      evalRunId
+      evalRunId,
+      evalDir
     };
     try {
       beforeAllOutput = await executeWorkspaceScript(suiteWorkspace.before_all, scriptContext);
@@ -15960,7 +15991,14 @@ async function runEvaluation(options) {
           answer: "",
           target: target.name,
           error: `Suite budget exceeded ($${cumulativeBudgetCost.toFixed(4)} / $${totalBudgetUsd.toFixed(4)})`,
-          budgetExceeded: true
+          budgetExceeded: true,
+          executionStatus: "execution_error",
+          failureStage: "setup",
+          failureReasonCode: "budget_exceeded",
+          executionError: {
+            message: `Suite budget exceeded ($${cumulativeBudgetCost.toFixed(4)} / $${totalBudgetUsd.toFixed(4)})`,
+            stage: "setup"
+          }
         };
         if (onProgress) {
           await onProgress({
@@ -16007,7 +16045,8 @@ async function runEvaluation(options) {
           suiteWorkspaceFile,
           streamCallbacks,
           typeRegistry,
-          repoManager
+          repoManager,
+          evalDir
         };
         let result = trials && trials.count > 1 ? await runEvalCaseWithTrials(runCaseOptions, trials) : await runEvalCase(runCaseOptions);
         if (totalBudgetUsd !== void 0) {
@@ -16076,7 +16115,9 @@ async function runEvaluation(options) {
         (now ?? (() => /* @__PURE__ */ new Date()))(),
         outcome.reason,
         promptInputs,
-        primaryProvider
+        primaryProvider,
+        "agent",
+        "provider_error"
       );
       results.push(errorResult);
       if (onResult) {
@@ -16088,7 +16129,8 @@ async function runEvaluation(options) {
     const scriptContext = {
       workspacePath: sharedWorkspacePath,
       testId: "__after_all__",
-      evalRunId
+      evalRunId,
+      evalDir
     };
     try {
       const afterAllOutput = await executeWorkspaceScript(
@@ -16218,7 +16260,14 @@ async function runBatchEvaluation(options) {
         availableTargets
       });
       if (providerError) {
-        result = { ...result, error: providerError };
+        result = {
+          ...result,
+          error: providerError,
+          executionStatus: "execution_error",
+          failureStage: "agent",
+          failureReasonCode: "provider_error",
+          executionError: { message: providerError, stage: "agent" }
+        };
       }
     } catch (error) {
       const errorResult = buildErrorResult(
@@ -16227,7 +16276,9 @@ async function runBatchEvaluation(options) {
         nowFn(),
         error,
         promptInputs,
-        provider
+        provider,
+        "evaluator",
+        "evaluator_error"
       );
       results.push(errorResult);
       if (onResult) {
@@ -16283,7 +16334,8 @@ async function runEvalCase(options) {
     sharedBaselineCommit,
     suiteWorkspaceFile,
     typeRegistry: providedTypeRegistry,
-    repoManager
+    repoManager,
+    evalDir
   } = options;
   const formattingMode = usesFileReferencePrompt(provider) ? "agent" : "lm";
   const promptInputs = await buildPromptInputs(evalCase, formattingMode);
@@ -16316,7 +16368,9 @@ async function runEvalCase(options) {
           nowFn(),
           new Error(`Failed to create workspace: ${message}`),
           promptInputs,
-          provider
+          provider,
+          "setup",
+          "template_error"
         );
       }
     }
@@ -16336,7 +16390,9 @@ async function runEvalCase(options) {
           nowFn(),
           new Error(`Failed to materialize repos: ${message}`),
           promptInputs,
-          provider
+          provider,
+          "repo_setup",
+          "clone_error"
         );
       }
     }
@@ -16346,7 +16402,8 @@ async function runEvalCase(options) {
         testId: evalCase.id,
         evalRunId: evalRunId ?? "",
         caseInput: evalCase.question,
-        caseMetadata: evalCase.metadata
+        caseMetadata: evalCase.metadata,
+        evalDir
       };
       try {
         beforeAllOutput = await executeWorkspaceScript(
@@ -16365,7 +16422,9 @@ async function runEvalCase(options) {
           nowFn(),
           new Error(`before_all script failed: ${message}`),
           promptInputs,
-          provider
+          provider,
+          "setup",
+          "script_error"
         );
       }
     }
@@ -16376,7 +16435,8 @@ async function runEvalCase(options) {
       testId: evalCase.id,
       evalRunId: evalRunId ?? "",
       caseInput: evalCase.question,
-      caseMetadata: evalCase.metadata
+      caseMetadata: evalCase.metadata,
+      evalDir
     };
     try {
       beforeEachOutput = await executeWorkspaceScript(
@@ -16391,7 +16451,9 @@ async function runEvalCase(options) {
         nowFn(),
         new Error(`before_each script failed: ${message}`),
         promptInputs,
-        provider
+        provider,
+        "setup",
+        "script_error"
       );
     }
   }
@@ -16432,7 +16494,9 @@ async function runEvalCase(options) {
         nowFn(),
         error,
         promptInputs,
-        provider
+        provider,
+        "agent",
+        "provider_error"
       );
       if (workspacePath) {
         if (forceCleanup) {
@@ -16451,7 +16515,9 @@ async function runEvalCase(options) {
       nowFn(),
       lastError ?? new Error("Provider did not return a response"),
       promptInputs,
-      provider
+      provider,
+      "agent",
+      "provider_error"
     );
     if (workspacePath) {
       if (forceCleanup) {
@@ -16507,7 +16573,8 @@ async function runEvalCase(options) {
       testId: evalCase.id,
       evalRunId: evalRunId ?? "",
       caseInput: evalCase.question,
-      caseMetadata: evalCase.metadata
+      caseMetadata: evalCase.metadata,
+      evalDir
     };
     try {
       afterEachOutput = await executeWorkspaceScript(
@@ -16543,7 +16610,18 @@ async function runEvalCase(options) {
       fileChanges,
       workspacePath
     });
-    const finalResult = providerError ? { ...result, error: providerError, beforeAllOutput, beforeEachOutput, afterEachOutput } : { ...result, beforeAllOutput, beforeEachOutput, afterEachOutput };
+    const executionStatus = providerError ? "execution_error" : classifyQualityStatus(result.score);
+    const finalResult = providerError ? {
+      ...result,
+      error: providerError,
+      executionStatus,
+      failureStage: "agent",
+      failureReasonCode: "provider_error",
+      executionError: { message: providerError, stage: "agent" },
+      beforeAllOutput,
+      beforeEachOutput,
+      afterEachOutput
+    } : { ...result, executionStatus, beforeAllOutput, beforeEachOutput, afterEachOutput };
     const isFailure = !!finalResult.error || finalResult.score < 0.5;
     if (workspacePath && !isSharedWorkspace) {
       if (forceCleanup) {
@@ -16564,7 +16642,9 @@ async function runEvalCase(options) {
       nowFn(),
       error,
       promptInputs,
-      provider
+      provider,
+      "evaluator",
+      "evaluator_error"
     );
     if (workspacePath && !isSharedWorkspace) {
       if (forceCleanup) {
@@ -16602,7 +16682,10 @@ async function runEvalCaseWithTrials(options, trialsConfig) {
       verdict: trialVerdict,
       scores: result.scores,
       error: result.error,
-      costUsd: trialCost
+      costUsd: trialCost,
+      executionStatus: result.executionStatus,
+      failureStage: result.failureStage,
+      failureReasonCode: result.failureReasonCode
     };
     trialResults.push(trial);
     if (trialCost !== void 0) {
@@ -16627,12 +16710,22 @@ async function runEvalCaseWithTrials(options, trialsConfig) {
     0
   );
   const baseResult = allResults[bestTrialIndex];
+  const hasOk = trialResults.some((t) => t.executionStatus === "ok");
+  const allExecutionError = trialResults.length > 0 && trialResults.every((t) => t.executionStatus === "execution_error");
+  const aggregateExecutionStatus = hasOk ? "ok" : allExecutionError ? "execution_error" : "quality_failure";
+  const aggregateFailureStage = aggregateExecutionStatus === "ok" ? void 0 : baseResult.failureStage;
+  const aggregateFailureReasonCode = aggregateExecutionStatus === "ok" ? void 0 : baseResult.failureReasonCode;
+  const aggregateExecutionError = aggregateExecutionStatus === "execution_error" ? baseResult.executionError : void 0;
   return {
     ...baseResult,
     score,
     trials: trialResults,
     aggregation,
-    costLimited: costLimited || void 0
+    costLimited: costLimited || void 0,
+    executionStatus: aggregateExecutionStatus,
+    failureStage: aggregateFailureStage,
+    failureReasonCode: aggregateFailureReasonCode,
+    executionError: aggregateExecutionError
   };
 }
 async function evaluateCandidate(options) {
@@ -16733,7 +16826,8 @@ async function evaluateCandidate(options) {
     scores,
     trace: trace2,
     output,
-    fileChanges
+    fileChanges,
+    executionStatus: classifyQualityStatus(score.score)
   };
 }
 async function runEvaluatorsForCase(options) {
@@ -17038,7 +17132,7 @@ async function invokeProvider(provider, options) {
     }
   }
 }
-function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs, provider) {
+function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs, provider, failureStage, failureReasonCode) {
   const message = error instanceof Error ? error.message : String(error);
   let agentRequest;
   let lmRequest;
@@ -17081,7 +17175,11 @@ function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs,
     target: targetName,
     requests,
     input,
-    error: message
+    error: message,
+    executionStatus: "execution_error",
+    failureStage,
+    failureReasonCode,
+    executionError: { message, stage: failureStage }
   };
 }
 function extractProviderError(response) {
@@ -17341,7 +17439,15 @@ var AgentVConfigSchema = import_zod6.z.object({
     /** Maximum retries on failure (default: 2) */
     maxRetries: import_zod6.z.number().int().min(0).optional(),
     /** Agent timeout in milliseconds (default: 120000) */
-    agentTimeoutMs: import_zod6.z.number().int().min(0).optional()
+    agentTimeoutMs: import_zod6.z.number().int().min(0).optional(),
+    /** Enable verbose logging */
+    verbose: import_zod6.z.boolean().optional(),
+    /** Write human-readable trace JSONL to this path (supports {timestamp} placeholder) */
+    traceFile: import_zod6.z.string().optional(),
+    /** Always keep temp workspaces after eval */
+    keepWorkspaces: import_zod6.z.boolean().optional(),
+    /** Write OTLP JSON trace to this path (supports {timestamp} placeholder) */
+    otelFile: import_zod6.z.string().optional()
   }).optional(),
   /** Output settings */
   output: import_zod6.z.object({