agentv 3.9.0 → 3.9.2

package/dist/{chunk-TXDPYXHY.js → chunk-OIVGGWJ3.js}

@@ -301,7 +301,7 @@ var require_dist = __commonJS({
   }
 });
 
-// ../../packages/core/dist/chunk-PC5TLJF6.js
+// ../../packages/core/dist/chunk-K7JCJIXA.js
 import { constants } from "node:fs";
 import { access, readFile } from "node:fs/promises";
 import path from "node:path";
@@ -419,7 +419,7 @@ __export(external_exports2, {
   void: () => voidType
 });
 
-// ../../packages/core/dist/chunk-PC5TLJF6.js
+// ../../packages/core/dist/chunk-K7JCJIXA.js
 import { readFile as readFile2 } from "node:fs/promises";
 import path3 from "node:path";
 import fg from "fast-glob";
@@ -14472,6 +14472,17 @@ function extractTargetsFromSuite(suite) {
   }
   return void 0;
 }
+function extractWorkersFromSuite(suite) {
+  const execution = suite.execution;
+  if (!execution || typeof execution !== "object" || Array.isArray(execution)) {
+    return void 0;
+  }
+  const workers = execution.workers;
+  if (typeof workers === "number" && Number.isInteger(workers) && workers >= 1 && workers <= 50) {
+    return workers;
+  }
+  return void 0;
+}
 function extractTargetsFromTestCase(testCase) {
   const execution = testCase.execution;
   if (!execution || typeof execution !== "object" || Array.isArray(execution)) {
@@ -16735,6 +16746,7 @@ async function loadTestSuite(evalFilePath, repoRoot, options) {
     tests,
     trials: extractTrialsConfig(parsed),
     targets: extractTargetsFromSuite(parsed),
+    workers: extractWorkersFromSuite(parsed),
     cacheConfig: extractCacheConfig(parsed),
     totalBudgetUsd: extractTotalBudgetUsd(parsed),
     ...metadata !== void 0 && { metadata },
@@ -23966,8 +23978,7 @@ ${context2.fileChanges}`;
     }
     const evaluatorRawRequest = {
       userPrompt,
-      systemPrompt,
-      target: graderProvider.targetName
+      systemPrompt
     };
     try {
       const { data, tokenUsage } = await this.runWithRetry({
@@ -23985,6 +23996,7 @@ ${context2.fileChanges}`;
         assertions,
         expectedAspectCount: Math.max(assertions.length, 1),
         evaluatorRawRequest,
+        graderTarget: graderProvider.targetName,
         tokenUsage
       };
     } catch (e) {
@@ -23996,7 +24008,8 @@ ${context2.fileChanges}`;
         verdict: "skip",
         assertions: [{ text: `Grader parse failure after 3 attempts: ${message}`, passed: false }],
         expectedAspectCount: 1,
-        evaluatorRawRequest
+        evaluatorRawRequest,
+        graderTarget: graderProvider.targetName
       };
     }
   }
@@ -24014,8 +24027,7 @@ ${context2.fileChanges}`;
     const systemPrompt = buildRubricOutputSchema();
     const evaluatorRawRequest = {
       userPrompt: prompt,
-      systemPrompt,
-      target: graderProvider.targetName
+      systemPrompt
     };
     try {
       const { data, tokenUsage } = await this.runWithRetry({
@@ -24032,6 +24044,7 @@ ${context2.fileChanges}`;
         assertions,
         expectedAspectCount: rubrics.length,
         evaluatorRawRequest,
+        graderTarget: graderProvider.targetName,
         tokenUsage
       };
     } catch (e) {
@@ -24043,7 +24056,8 @@ ${context2.fileChanges}`;
         verdict: "skip",
         assertions: [{ text: `Grader parse failure after 3 attempts: ${message}`, passed: false }],
         expectedAspectCount: rubrics.length,
-        evaluatorRawRequest
+        evaluatorRawRequest,
+        graderTarget: graderProvider.targetName
       };
     }
   }
@@ -24056,8 +24070,7 @@ ${context2.fileChanges}`;
     const systemPrompt = buildScoreRangeOutputSchema();
     const evaluatorRawRequest = {
       userPrompt: prompt,
-      systemPrompt,
-      target: graderProvider.targetName
+      systemPrompt
     };
     try {
       const { data, tokenUsage } = await this.runWithRetry({
@@ -24074,6 +24087,7 @@ ${context2.fileChanges}`;
         assertions,
         expectedAspectCount: rubrics.length,
         evaluatorRawRequest,
+        graderTarget: graderProvider.targetName,
         details,
         tokenUsage
       };
@@ -24086,7 +24100,8 @@ ${context2.fileChanges}`;
         verdict: "skip",
         assertions: [{ text: `Grader parse failure after 3 attempts: ${message}`, passed: false }],
         expectedAspectCount: rubrics.length,
-        evaluatorRawRequest
+        evaluatorRawRequest,
+        graderTarget: graderProvider.targetName
       };
     }
   }
@@ -24118,7 +24133,6 @@ ${context2.fileChanges}`;
       mode: "built-in",
       systemPrompt,
       userPrompt,
-      target: graderProvider.targetName,
       maxSteps: this.maxSteps
     };
     try {
@@ -24136,7 +24150,13 @@ ${context2.fileChanges}`;
         steps: steps.length,
         tool_calls: toolCallCount
       };
-      return this.parseAgentResult(text2, rubrics, evaluatorRawRequest, details);
+      return this.parseAgentResult(
+        text2,
+        rubrics,
+        evaluatorRawRequest,
+        details,
+        graderProvider.targetName
+      );
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       return {
@@ -24145,6 +24165,7 @@ ${context2.fileChanges}`;
         assertions: [{ text: `llm-grader built-in evaluation failed: ${message}`, passed: false }],
         expectedAspectCount: 1,
         evaluatorRawRequest,
+        graderTarget: graderProvider.targetName,
         details: { mode: "built-in", error: message }
       };
     }
@@ -24197,6 +24218,7 @@ ${context2.fileChanges}`;
           ],
           expectedAspectCount: 1,
           evaluatorRawRequest,
+          graderTarget: provider.targetName,
           details: { mode: modeLabel, grader_target: provider.targetName }
         };
       }
@@ -24206,7 +24228,13 @@ ${context2.fileChanges}`;
         mode: modeLabel,
         grader_target: provider.targetName
       };
-      return this.parseAgentResult(assistantContent, rubrics, evaluatorRawRequest, details);
+      return this.parseAgentResult(
+        assistantContent,
+        rubrics,
+        evaluatorRawRequest,
+        details,
+        provider.targetName
+      );
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       return {
@@ -24217,6 +24245,7 @@ ${context2.fileChanges}`;
         ],
         expectedAspectCount: 1,
         evaluatorRawRequest,
+        graderTarget: provider.targetName,
         details: {
           mode: modeLabel,
           grader_target: provider.targetName,
@@ -24361,7 +24390,7 @@ ${outputSchema2}`;
    * Parse the agent's response text into an EvaluationScore.
    * Supports both freeform and rubric modes.
    */
-  parseAgentResult(text2, rubrics, evaluatorRawRequest, details) {
+  parseAgentResult(text2, rubrics, evaluatorRawRequest, details, graderTarget) {
     try {
       const parsed = parseJsonFromText(text2);
       if (rubrics && rubrics.length > 0) {
@@ -24373,6 +24402,7 @@ ${outputSchema2}`;
           assertions: assertions2,
           expectedAspectCount: rubrics.length,
           evaluatorRawRequest,
+          graderTarget,
           details
         };
       }
@@ -24385,6 +24415,7 @@ ${outputSchema2}`;
         assertions,
         expectedAspectCount: Math.max(assertions.length, 1),
         evaluatorRawRequest,
+        graderTarget,
         details
       };
     } catch {
@@ -24399,6 +24430,7 @@ ${outputSchema2}`;
         ],
         expectedAspectCount: 1,
         evaluatorRawRequest,
+        graderTarget,
         details
       };
     }
@@ -28331,14 +28363,22 @@ async function runEvaluation(options) {
   const usePool = poolEnabled !== false && !!suiteWorkspace?.repos?.length && !isPerTestIsolation && !useStaticWorkspace;
   const resolvedRetainOnSuccess = retainOnSuccess ?? (keepWorkspaces ? "keep" : "cleanup");
   const resolvedRetainOnFailure = retainOnFailure ?? (cleanupWorkspaces ? "cleanup" : "keep");
-  const requestedWorkers = options.maxConcurrency ?? target.workers ?? 1;
-  const workers = hasSharedWorkspace && !usePool ? 1 : requestedWorkers;
+  const workers = options.maxConcurrency ?? target.workers ?? 1;
   setupLog(
-    `sharedWorkspace=${hasSharedWorkspace} perTestIsolation=${isPerTestIsolation} usePool=${usePool} requestedWorkers=${requestedWorkers} effectiveWorkers=${workers}`
+    `sharedWorkspace=${hasSharedWorkspace} perTestIsolation=${isPerTestIsolation} usePool=${usePool} workers=${workers}`
   );
-  if (hasSharedWorkspace && !usePool && requestedWorkers > 1) {
+  if (hasSharedWorkspace && !usePool && workers > 1) {
     console.warn(
-      `Warning: Shared workspace requires sequential execution. Overriding workers from ${requestedWorkers} to 1.`
+      [
+        `Warning: This eval uses a shared workspace with ${workers} workers.`,
+        "If the agent under test makes file edits, concurrent runs may corrupt each other.",
+        "To limit concurrency, add this to your eval YAML:",
+        "",
+        "  execution:",
+        "    workers: 1",
+        "",
+        "Or pass --workers 1 on the command line."
+      ].join("\n")
     );
   }
   const limit = pLimit(workers);
@@ -28621,7 +28661,8 @@ async function runEvaluation(options) {
             streamCallbacks,
             typeRegistry,
             repoManager,
-            evalDir
+            evalDir,
+            verbose
           };
           let result = trials && trials.count > 1 ? await runEvalCaseWithTrials(runCaseOptions, trials) : await runEvalCase(runCaseOptions);
           if (totalBudgetUsd !== void 0) {
@@ -28701,7 +28742,8 @@ async function runEvaluation(options) {
           promptInputs,
           primaryProvider,
           "agent",
-          "provider_error"
+          "provider_error",
+          verbose
         );
         results.push(errorResult);
         if (onResult) {
@@ -28774,6 +28816,7 @@ async function runBatchEvaluation(options) {
     nowFn,
     onProgress,
     onResult,
+    verbose,
     resolveGraderProvider,
     agentTimeoutMs,
     targetResolver,
@@ -28861,7 +28904,8 @@ async function runBatchEvaluation(options) {
         startTime,
         endTime,
         targetResolver,
-        availableTargets
+        availableTargets,
+        verbose
       });
       if (providerError) {
         result = {
@@ -28882,7 +28926,8 @@ async function runBatchEvaluation(options) {
         promptInputs,
         provider,
         "evaluator",
-        "evaluator_error"
+        "evaluator_error",
+        verbose
       );
       results.push(errorResult);
       if (onResult) {
@@ -28945,7 +28990,8 @@ async function runEvalCase(options) {
     suiteWorkspaceFile,
     typeRegistry: providedTypeRegistry,
     repoManager,
-    evalDir
+    evalDir,
+    verbose
   } = options;
   const setupDebug = process.env.AGENTV_SETUP_DEBUG === "1";
   const formattingMode = usesFileReferencePrompt(provider) ? "agent" : "lm";
@@ -28982,7 +29028,8 @@ async function runEvalCase(options) {
           promptInputs,
           provider,
           "setup",
-          "template_error"
+          "template_error",
+          verbose
         );
       }
       if (caseWorkspaceFile && workspacePath) {
@@ -29011,7 +29058,8 @@ async function runEvalCase(options) {
           promptInputs,
           provider,
           "repo_setup",
-          "local_path_not_found"
+          "local_path_not_found",
+          verbose
         );
       }
     }
@@ -29037,7 +29085,8 @@ async function runEvalCase(options) {
           promptInputs,
           provider,
           "repo_setup",
-          "clone_error"
+          "clone_error",
+          verbose
         );
       }
     }
@@ -29063,7 +29112,8 @@ async function runEvalCase(options) {
               promptInputs,
               provider,
               "setup",
-              "file_copy_error"
+              "file_copy_error",
+              verbose
             );
           }
         }
@@ -29108,7 +29158,8 @@ async function runEvalCase(options) {
           promptInputs,
           provider,
           "setup",
-          "script_error"
+          "script_error",
+          verbose
         );
       }
     }
@@ -29139,7 +29190,8 @@ async function runEvalCase(options) {
         promptInputs,
         provider,
         "setup",
-        "script_error"
+        "script_error",
+        verbose
       );
     }
   }
@@ -29183,7 +29235,8 @@ async function runEvalCase(options) {
         promptInputs,
         provider,
         "agent",
-        "provider_error"
+        "provider_error",
+        verbose
       );
       if (workspacePath) {
         if (forceCleanup) {
@@ -29204,7 +29257,8 @@ async function runEvalCase(options) {
       promptInputs,
       provider,
       "agent",
-      "provider_error"
+      "provider_error",
+      verbose
     );
     if (workspacePath) {
       if (forceCleanup) {
@@ -29299,7 +29353,8 @@ async function runEvalCase(options) {
       targetResolver,
       availableTargets,
       fileChanges,
-      workspacePath
+      workspacePath,
+      verbose
     });
     const totalDurationMs = Date.now() - caseStartMs;
     const graderTokens = aggregateEvaluatorTokenUsage(result.scores);
@@ -29354,7 +29409,8 @@ async function runEvalCase(options) {
       promptInputs,
       provider,
       "evaluator",
-      "evaluator_error"
+      "evaluator_error",
+      verbose
     );
     if (workspacePath && !isSharedWorkspace) {
       if (forceCleanup || (retainOnFailure ?? "keep") === "cleanup") {
@@ -29496,7 +29552,7 @@ async function evaluateCandidate(options) {
   let lmRequest;
   if (isAgentProvider(provider)) {
     agentRequest = {
-      question: promptInputs.question
+      ...options.verbose ? { input: promptInputs.question } : {}
     };
   } else {
     if (promptInputs.chatPrompt) {
@@ -29510,8 +29566,9 @@ async function evaluateCandidate(options) {
     }
   }
   const evaluatorRequest = scores ? void 0 : score.evaluatorRawRequest;
-  const requests = agentRequest || lmRequest || evaluatorRequest ? {
-    ...agentRequest ? { agent: agentRequest } : {},
+  const effectiveAgentRequest = agentRequest && Object.keys(agentRequest).length > 0 ? agentRequest : void 0;
+  const requests = effectiveAgentRequest || lmRequest || evaluatorRequest ? {
+    ...effectiveAgentRequest ? { agent: effectiveAgentRequest } : {},
     ...lmRequest ? { lm: lmRequest } : {},
     ...evaluatorRequest ? { evaluator: evaluatorRequest } : {}
   } : void 0;
@@ -29531,9 +29588,9 @@ async function evaluateCandidate(options) {
     endTime,
     requests,
     input,
+    output: output ?? [{ role: "assistant", content: candidate }],
     scores,
     trace: trace2,
-    output: output ?? [{ role: "assistant", content: candidate }],
     fileChanges,
     executionStatus: classifyQualityStatus(score.score)
   };
@@ -29699,6 +29756,7 @@ async function runEvaluatorList(options) {
         verdict: score2.verdict,
         assertions: score2.assertions,
         input: score2.evaluatorRawRequest,
+        target: score2.graderTarget,
         details: score2.details,
         scores: mapChildResults(score2.scores),
         tokenUsage: score2.tokenUsage,
@@ -29838,13 +29896,13 @@ async function invokeProvider(provider, options) {
     }
   }
 }
-function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs, provider, failureStage, failureReasonCode) {
+function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs, provider, failureStage, failureReasonCode, verbose) {
   const message = error instanceof Error ? error.message : String(error);
   let agentRequest;
   let lmRequest;
   if (isAgentProvider(provider)) {
     agentRequest = {
-      question: promptInputs.question,
+      ...verbose ? { input: promptInputs.question } : {},
       error: message
     };
   } else {
@@ -29872,10 +29930,10 @@ function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs,
     conversationId: evalCase.conversation_id,
     score: 0,
     assertions: [{ text: `Error: ${message}`, passed: false }],
-    output: [{ role: "assistant", content: `Error occurred: ${message}` }],
     target: targetName,
     requests,
     input,
+    output: [{ role: "assistant", content: `Error occurred: ${message}` }],
     error: message,
     executionStatus: "execution_error",
     failureStage,
@@ -30878,6 +30936,7 @@ export {
   loadConfig,
   extractTargetFromSuite,
   extractTargetsFromSuite,
+  extractWorkersFromSuite,
   extractTargetsFromTestCase,
   extractTrialsConfig,
   extractCacheConfig,
@@ -30992,4 +31051,4 @@ export {
   OtelStreamingObserver,
   createAgentKernel
 };
-//# sourceMappingURL=chunk-TXDPYXHY.js.map
+//# sourceMappingURL=chunk-OIVGGWJ3.js.map