@agentv/core 3.9.1 → 3.10.0

package/dist/index.d.cts

@@ -1196,6 +1196,8 @@ interface EvaluatorResult {
     readonly assertions: readonly AssertionEntry[];
     readonly rawRequest?: JsonObject;
     readonly input?: JsonObject;
+    /** Target name used for grading (e.g., the LLM provider name). */
+    readonly target?: string;
     readonly scores?: readonly EvaluatorResult[];
     /** Optional structured details from code graders (e.g., TP/TN/FP/FN counts). */
     readonly details?: JsonObject;
@@ -2057,6 +2059,8 @@ interface EvaluationScore {
     readonly details?: JsonObject;
     /** Token usage from LLM calls made by this evaluator (optional). */
     readonly tokenUsage?: TokenUsage;
+    /** Target name used for grading (e.g., the LLM provider). */
+    readonly graderTarget?: string;
 }
 interface ChildEvaluatorResult {
     readonly name: string;
@@ -2660,6 +2664,8 @@ interface RunEvalCaseOptions {
     readonly repoManager?: RepoManager;
     /** Directory containing the eval YAML file. Used as default cwd for workspace scripts. */
     readonly evalDir?: string;
+    /** Include verbose request details in results (e.g. agent input text) */
+    readonly verbose?: boolean;
 }
 interface ProgressEvent {
     readonly workerId: number;

package/dist/index.d.ts

@@ -1196,6 +1196,8 @@ interface EvaluatorResult {
     readonly assertions: readonly AssertionEntry[];
     readonly rawRequest?: JsonObject;
     readonly input?: JsonObject;
+    /** Target name used for grading (e.g., the LLM provider name). */
+    readonly target?: string;
     readonly scores?: readonly EvaluatorResult[];
     /** Optional structured details from code graders (e.g., TP/TN/FP/FN counts). */
     readonly details?: JsonObject;
@@ -2057,6 +2059,8 @@ interface EvaluationScore {
     readonly details?: JsonObject;
     /** Token usage from LLM calls made by this evaluator (optional). */
     readonly tokenUsage?: TokenUsage;
+    /** Target name used for grading (e.g., the LLM provider). */
+    readonly graderTarget?: string;
 }
 interface ChildEvaluatorResult {
     readonly name: string;
@@ -2660,6 +2664,8 @@ interface RunEvalCaseOptions {
     readonly repoManager?: RepoManager;
     /** Directory containing the eval YAML file. Used as default cwd for workspace scripts. */
     readonly evalDir?: string;
+    /** Include verbose request details in results (e.g. agent input text) */
+    readonly verbose?: boolean;
 }
 interface ProgressEvent {
     readonly workerId: number;

package/dist/index.js

@@ -19,7 +19,7 @@ import {
   readTextFile,
   resolveFileReference,
   resolveTargetDefinition
-} from "./chunk-PC5TLJF6.js";
+} from "./chunk-K7JCJIXA.js";
 import {
   AgentvProvider
 } from "./chunk-W5YDZWT4.js";
@@ -6112,11 +6112,7 @@ var CopilotCliProvider = class {
           }
         }
         if (sessionUpdate === "usage_update") {
-          if (tokenUsage) {
-            tokenUsage = { input: update.used, output: tokenUsage.output };
-          } else {
-            tokenUsage = { input: update.used, output: 0 };
-          }
+          tokenUsage = { input: update.used, output: 0 };
           if (update.cost && update.cost.currency === "USD") {
             costUsd = (costUsd ?? 0) + update.cost.amount;
           }
@@ -6150,21 +6146,32 @@ var CopilotCliProvider = class {
         sessionId: session.sessionId,
         prompt: promptMessages
       });
+      let promptResponse;
       if (request.signal) {
         const abortHandler = () => {
           killProcess(agentProcess);
         };
         request.signal.addEventListener("abort", abortHandler, { once: true });
         try {
-          await this.raceWithTimeout(sendPromise, agentProcess);
+          promptResponse = await this.raceWithTimeout(sendPromise, agentProcess);
         } finally {
           request.signal.removeEventListener("abort", abortHandler);
         }
       } else {
-        await this.raceWithTimeout(sendPromise, agentProcess);
+        promptResponse = await this.raceWithTimeout(sendPromise, agentProcess);
       }
       const endTime = (/* @__PURE__ */ new Date()).toISOString();
       const durationMs = Date.now() - startMs;
+      const responseUsage = promptResponse.usage;
+      if (responseUsage && responseUsage.totalTokens > 0) {
+        tokenUsage = {
+          input: responseUsage.inputTokens,
+          output: responseUsage.outputTokens,
+          ...responseUsage.thoughtTokens != null ? { reasoning: responseUsage.thoughtTokens } : {},
+          ...responseUsage.cachedReadTokens != null ? { cached: responseUsage.cachedReadTokens } : {}
+        };
+        request.streamCallbacks?.onLlmCallEnd?.("copilot", tokenUsage);
+      }
       const rejectedCalls = completedToolCalls.filter((tc) => {
         const out = tc.output;
         return out && (out.code === "rejected" || out.code === "denied");
@@ -6222,8 +6229,7 @@ var CopilotCliProvider = class {
   async raceWithTimeout(sendPromise, agentProcess) {
     const timeoutMs = this.config.timeoutMs;
     if (!timeoutMs) {
-      await sendPromise;
-      return;
+      return sendPromise;
     }
     let timer;
     const timeoutPromise = new Promise((_, reject) => {
@@ -6234,7 +6240,7 @@ var CopilotCliProvider = class {
       timer.unref?.();
     });
     try {
-      await Promise.race([sendPromise, timeoutPromise]);
+      return await Promise.race([sendPromise, timeoutPromise]);
     } finally {
       if (timer) clearTimeout(timer);
     }
@@ -9287,7 +9293,7 @@ async function readTargetDefinitions(filePath) {
     throw new Error(`targets.yaml not found at ${absolutePath}`);
   }
   const raw = await readFile9(absolutePath, "utf8");
-  const parsed = parse4(raw);
+  const parsed = interpolateEnv(parse4(raw), process.env);
   if (!isRecord(parsed)) {
     throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with a 'targets' field`);
   }
@@ -10172,8 +10178,7 @@ ${context.fileChanges}`;
     }
     const evaluatorRawRequest = {
       userPrompt,
-      systemPrompt,
-      target: graderProvider.targetName
+      systemPrompt
     };
     try {
       const { data, tokenUsage } = await this.runWithRetry({
@@ -10191,6 +10196,7 @@ ${context.fileChanges}`;
         assertions,
         expectedAspectCount: Math.max(assertions.length, 1),
         evaluatorRawRequest,
+        graderTarget: graderProvider.targetName,
         tokenUsage
       };
     } catch (e) {
@@ -10202,7 +10208,8 @@ ${context.fileChanges}`;
         verdict: "skip",
         assertions: [{ text: `Grader parse failure after 3 attempts: ${message}`, passed: false }],
         expectedAspectCount: 1,
-        evaluatorRawRequest
+        evaluatorRawRequest,
+        graderTarget: graderProvider.targetName
       };
     }
   }
@@ -10220,8 +10227,7 @@ ${context.fileChanges}`;
     const systemPrompt = buildRubricOutputSchema();
     const evaluatorRawRequest = {
       userPrompt: prompt,
-      systemPrompt,
-      target: graderProvider.targetName
+      systemPrompt
     };
     try {
       const { data, tokenUsage } = await this.runWithRetry({
@@ -10238,6 +10244,7 @@ ${context.fileChanges}`;
         assertions,
         expectedAspectCount: rubrics.length,
         evaluatorRawRequest,
+        graderTarget: graderProvider.targetName,
         tokenUsage
       };
     } catch (e) {
@@ -10249,7 +10256,8 @@ ${context.fileChanges}`;
         verdict: "skip",
         assertions: [{ text: `Grader parse failure after 3 attempts: ${message}`, passed: false }],
         expectedAspectCount: rubrics.length,
-        evaluatorRawRequest
+        evaluatorRawRequest,
+        graderTarget: graderProvider.targetName
       };
     }
   }
@@ -10262,8 +10270,7 @@ ${context.fileChanges}`;
     const systemPrompt = buildScoreRangeOutputSchema();
     const evaluatorRawRequest = {
       userPrompt: prompt,
-      systemPrompt,
-      target: graderProvider.targetName
+      systemPrompt
     };
     try {
       const { data, tokenUsage } = await this.runWithRetry({
@@ -10280,6 +10287,7 @@ ${context.fileChanges}`;
         assertions,
         expectedAspectCount: rubrics.length,
         evaluatorRawRequest,
+        graderTarget: graderProvider.targetName,
         details,
         tokenUsage
       };
@@ -10292,7 +10300,8 @@ ${context.fileChanges}`;
         verdict: "skip",
         assertions: [{ text: `Grader parse failure after 3 attempts: ${message}`, passed: false }],
         expectedAspectCount: rubrics.length,
-        evaluatorRawRequest
+        evaluatorRawRequest,
+        graderTarget: graderProvider.targetName
       };
     }
   }
@@ -10324,7 +10333,6 @@ ${context.fileChanges}`;
       mode: "built-in",
       systemPrompt,
       userPrompt,
-      target: graderProvider.targetName,
       maxSteps: this.maxSteps
     };
     try {
@@ -10342,7 +10350,13 @@ ${context.fileChanges}`;
         steps: steps.length,
         tool_calls: toolCallCount
       };
-      return this.parseAgentResult(text, rubrics, evaluatorRawRequest, details);
+      return this.parseAgentResult(
+        text,
+        rubrics,
+        evaluatorRawRequest,
+        details,
+        graderProvider.targetName
+      );
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       return {
@@ -10351,6 +10365,7 @@ ${context.fileChanges}`;
         assertions: [{ text: `llm-grader built-in evaluation failed: ${message}`, passed: false }],
         expectedAspectCount: 1,
         evaluatorRawRequest,
+        graderTarget: graderProvider.targetName,
         details: { mode: "built-in", error: message }
       };
     }
@@ -10403,6 +10418,7 @@ ${context.fileChanges}`;
           ],
           expectedAspectCount: 1,
           evaluatorRawRequest,
+          graderTarget: provider.targetName,
           details: { mode: modeLabel, grader_target: provider.targetName }
         };
       }
@@ -10412,7 +10428,13 @@ ${context.fileChanges}`;
         mode: modeLabel,
         grader_target: provider.targetName
       };
-      return this.parseAgentResult(assistantContent, rubrics, evaluatorRawRequest, details);
+      return this.parseAgentResult(
+        assistantContent,
+        rubrics,
+        evaluatorRawRequest,
+        details,
+        provider.targetName
+      );
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       return {
@@ -10423,6 +10445,7 @@ ${context.fileChanges}`;
         ],
         expectedAspectCount: 1,
         evaluatorRawRequest,
+        graderTarget: provider.targetName,
         details: {
           mode: modeLabel,
           grader_target: provider.targetName,
@@ -10567,7 +10590,7 @@ ${outputSchema}`;
    * Parse the agent's response text into an EvaluationScore.
    * Supports both freeform and rubric modes.
    */
-  parseAgentResult(text, rubrics, evaluatorRawRequest, details) {
+  parseAgentResult(text, rubrics, evaluatorRawRequest, details, graderTarget) {
     try {
       const parsed = parseJsonFromText(text);
       if (rubrics && rubrics.length > 0) {
@@ -10579,6 +10602,7 @@ ${outputSchema}`;
           assertions: assertions2,
           expectedAspectCount: rubrics.length,
           evaluatorRawRequest,
+          graderTarget,
           details
         };
       }
@@ -10591,6 +10615,7 @@ ${outputSchema}`;
         assertions,
         expectedAspectCount: Math.max(assertions.length, 1),
         evaluatorRawRequest,
+        graderTarget,
         details
       };
     } catch {
@@ -10605,6 +10630,7 @@ ${outputSchema}`;
         ],
         expectedAspectCount: 1,
         evaluatorRawRequest,
+        graderTarget,
         details
       };
     }
@@ -14916,7 +14942,8 @@ async function runEvaluation(options) {
             streamCallbacks,
             typeRegistry,
             repoManager,
-            evalDir
+            evalDir,
+            verbose
           };
           let result = trials && trials.count > 1 ? await runEvalCaseWithTrials(runCaseOptions, trials) : await runEvalCase(runCaseOptions);
           if (totalBudgetUsd !== void 0) {
@@ -14996,7 +15023,8 @@ async function runEvaluation(options) {
           promptInputs,
           primaryProvider,
           "agent",
-          "provider_error"
+          "provider_error",
+          verbose
         );
         results.push(errorResult);
         if (onResult) {
@@ -15069,6 +15097,7 @@ async function runBatchEvaluation(options) {
     nowFn,
     onProgress,
     onResult,
+    verbose,
     resolveGraderProvider,
     agentTimeoutMs,
     targetResolver,
@@ -15156,7 +15185,8 @@ async function runBatchEvaluation(options) {
         startTime,
         endTime,
         targetResolver,
-        availableTargets
+        availableTargets,
+        verbose
       });
       if (providerError) {
         result = {
@@ -15177,7 +15207,8 @@ async function runBatchEvaluation(options) {
         promptInputs,
         provider,
         "evaluator",
-        "evaluator_error"
+        "evaluator_error",
+        verbose
       );
       results.push(errorResult);
       if (onResult) {
@@ -15240,7 +15271,8 @@ async function runEvalCase(options) {
     suiteWorkspaceFile,
     typeRegistry: providedTypeRegistry,
     repoManager,
-    evalDir
+    evalDir,
+    verbose
   } = options;
   const setupDebug = process.env.AGENTV_SETUP_DEBUG === "1";
   const formattingMode = usesFileReferencePrompt(provider) ? "agent" : "lm";
@@ -15277,7 +15309,8 @@ async function runEvalCase(options) {
           promptInputs,
           provider,
           "setup",
-          "template_error"
+          "template_error",
+          verbose
         );
       }
       if (caseWorkspaceFile && workspacePath) {
@@ -15306,7 +15339,8 @@ async function runEvalCase(options) {
           promptInputs,
           provider,
           "repo_setup",
-          "local_path_not_found"
+          "local_path_not_found",
+          verbose
         );
       }
     }
@@ -15332,7 +15366,8 @@ async function runEvalCase(options) {
           promptInputs,
           provider,
           "repo_setup",
-          "clone_error"
+          "clone_error",
+          verbose
         );
       }
     }
@@ -15358,7 +15393,8 @@ async function runEvalCase(options) {
               promptInputs,
               provider,
               "setup",
-              "file_copy_error"
+              "file_copy_error",
+              verbose
             );
           }
         }
@@ -15403,7 +15439,8 @@ async function runEvalCase(options) {
           promptInputs,
           provider,
           "setup",
-          "script_error"
+          "script_error",
+          verbose
         );
       }
     }
@@ -15434,7 +15471,8 @@ async function runEvalCase(options) {
         promptInputs,
         provider,
         "setup",
-        "script_error"
+        "script_error",
+        verbose
       );
     }
   }
@@ -15478,7 +15516,8 @@ async function runEvalCase(options) {
         promptInputs,
         provider,
         "agent",
-        "provider_error"
+        "provider_error",
+        verbose
       );
       if (workspacePath) {
         if (forceCleanup) {
@@ -15499,7 +15538,8 @@ async function runEvalCase(options) {
       promptInputs,
       provider,
       "agent",
-      "provider_error"
+      "provider_error",
+      verbose
     );
     if (workspacePath) {
       if (forceCleanup) {
@@ -15594,7 +15634,8 @@ async function runEvalCase(options) {
       targetResolver,
       availableTargets,
       fileChanges,
-      workspacePath
+      workspacePath,
+      verbose
     });
     const totalDurationMs = Date.now() - caseStartMs;
     const graderTokens = aggregateEvaluatorTokenUsage(result.scores);
@@ -15649,7 +15690,8 @@ async function runEvalCase(options) {
       promptInputs,
       provider,
       "evaluator",
-      "evaluator_error"
+      "evaluator_error",
+      verbose
     );
     if (workspacePath && !isSharedWorkspace) {
       if (forceCleanup || (retainOnFailure ?? "keep") === "cleanup") {
@@ -15791,7 +15833,7 @@ async function evaluateCandidate(options) {
   let lmRequest;
   if (isAgentProvider(provider)) {
     agentRequest = {
-      question: promptInputs.question
+      ...options.verbose ? { input: promptInputs.question } : {}
     };
   } else {
     if (promptInputs.chatPrompt) {
@@ -15805,8 +15847,9 @@ async function evaluateCandidate(options) {
     }
   }
   const evaluatorRequest = scores ? void 0 : score.evaluatorRawRequest;
-  const requests = agentRequest || lmRequest || evaluatorRequest ? {
-    ...agentRequest ? { agent: agentRequest } : {},
+  const effectiveAgentRequest = agentRequest && Object.keys(agentRequest).length > 0 ? agentRequest : void 0;
+  const requests = effectiveAgentRequest || lmRequest || evaluatorRequest ? {
+    ...effectiveAgentRequest ? { agent: effectiveAgentRequest } : {},
     ...lmRequest ? { lm: lmRequest } : {},
     ...evaluatorRequest ? { evaluator: evaluatorRequest } : {}
   } : void 0;
@@ -15826,9 +15869,9 @@ async function evaluateCandidate(options) {
     endTime,
     requests,
     input,
+    output: output ?? [{ role: "assistant", content: candidate }],
     scores,
     trace,
-    output: output ?? [{ role: "assistant", content: candidate }],
     fileChanges,
     executionStatus: classifyQualityStatus(score.score)
   };
@@ -15994,6 +16037,7 @@ async function runEvaluatorList(options) {
         verdict: score2.verdict,
         assertions: score2.assertions,
         input: score2.evaluatorRawRequest,
+        target: score2.graderTarget,
         details: score2.details,
         scores: mapChildResults(score2.scores),
         tokenUsage: score2.tokenUsage,
@@ -16133,13 +16177,13 @@ async function invokeProvider(provider, options) {
     }
   }
 }
-function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs, provider, failureStage, failureReasonCode) {
+function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs, provider, failureStage, failureReasonCode, verbose) {
   const message = error instanceof Error ? error.message : String(error);
   let agentRequest;
   let lmRequest;
   if (isAgentProvider(provider)) {
     agentRequest = {
-      question: promptInputs.question,
+      ...verbose ? { input: promptInputs.question } : {},
       error: message
     };
   } else {
@@ -16167,10 +16211,10 @@ function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs,
     conversationId: evalCase.conversation_id,
     score: 0,
     assertions: [{ text: `Error: ${message}`, passed: false }],
-    output: [{ role: "assistant", content: `Error occurred: ${message}` }],
     target: targetName,
     requests,
     input,
+    output: [{ role: "assistant", content: `Error occurred: ${message}` }],
     error: message,
     executionStatus: "execution_error",
     failureStage,