agentv 3.7.0 → 3.9.0

package/README.md

@@ -164,7 +164,7 @@ For large-scale evaluations, AgentV supports JSONL (JSON Lines) format as an alt
 Optional sidecar YAML metadata file (`dataset.eval.yaml` alongside `dataset.jsonl`):
 ```yaml
 description: Math evaluation dataset
-dataset: math-tests
+name: math-tests
 execution:
   target: azure-llm
 assertions:

package/dist/{chunk-7YS6YNJZ.js → chunk-GC5P5HHZ.js}

@@ -8,8 +8,10 @@ import {
   buildSearchRoots,
   ensureVSCodeSubagents,
   findGitRoot,
+  interpolateEnv,
   isEvaluatorKind,
   listTargetNames,
+  loadCasesFromFile,
   loadConfig,
   loadTestSuite,
   loadTsConfig,
@@ -25,12 +27,12 @@ import {
   subscribeToCopilotCliLogEntries,
   subscribeToCopilotSdkLogEntries,
   subscribeToPiLogEntries
-} from "./chunk-XGG64VIY.js";
+} from "./chunk-TXDPYXHY.js";
 
 // package.json
 var package_default = {
   name: "agentv",
-  version: "3.7.0",
+  version: "3.9.0",
   description: "CLI entry point for AgentV",
   type: "module",
   repository: {
@@ -1326,9 +1328,9 @@ var SCRIPT = `
     /* input / output */
     h+='<div class="detail-grid">';
     if(r.input!=null){
-      h+='<div class="detail-block"><h4>Input</h4><pre class="detail-pre">'+esc(typeof r.input==="string"?r.input:JSON.stringify(r.input,null,2))+"</pre></div>";
+      h+='<div class="detail-block"><h4>Input</h4><pre class="detail-pre">'+esc(JSON.stringify(r.input,null,2))+"</pre></div>";
     }
-    h+='<div class="detail-block"><h4>Output</h4><pre class="detail-pre">'+esc(r.outputText||"")+"</pre></div>";
+    h+='<div class="detail-block"><h4>Output</h4><pre class="detail-pre">'+esc(r.output?JSON.stringify(r.output,null,2):"")+"</pre></div>";
     h+="</div>";
 
     /* evaluator results */
@@ -1522,7 +1524,7 @@ var JunitWriter = class _JunitWriter {
     this.closed = true;
     const grouped = /* @__PURE__ */ new Map();
     for (const result of this.results) {
-      const suite = result.dataset ?? "default";
+      const suite = result.eval_set ?? "default";
       const existing = grouped.get(suite);
       if (existing) {
         existing.push(result);
@@ -2186,7 +2188,7 @@ async function validateEvalFile(filePath) {
   let parsed;
   try {
     const content = await readFile22(absolutePath, "utf8");
-    parsed = parse2(content);
+    parsed = interpolateEnv(parse2(content), process.env);
   } catch (error) {
     errors.push({
       severity: "error",
@@ -2249,6 +2251,31 @@ async function validateEvalFile(filePath) {
   }
   if (typeof cases === "string") {
     validateTestsStringPath(cases, absolutePath, errors);
+    await validateWorkspaceConfig(parsed.workspace, absolutePath, errors, "workspace");
+    const ext = path22.extname(cases).toLowerCase();
+    if (VALID_TEST_FILE_EXTENSIONS.has(ext)) {
+      const externalCasesPath = path22.resolve(path22.dirname(absolutePath), cases);
+      try {
+        const externalCases = await loadCasesFromFile(externalCasesPath);
+        for (let i = 0; i < externalCases.length; i++) {
+          const externalCase = externalCases[i];
+          await validateWorkspaceConfig(
+            externalCase.workspace,
+            absolutePath,
+            errors,
+            `tests[${i}].workspace`
+          );
+        }
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        errors.push({
+          severity: "error",
+          filePath: absolutePath,
+          location: "tests",
+          message
+        });
+      }
+    }
     return {
       valid: errors.filter((e) => e.severity === "error").length === 0,
       filePath: absolutePath,
@@ -2356,10 +2383,14 @@ async function validateEvalFile(filePath) {
     if (assertField !== void 0) {
       validateAssertArray(assertField, location, absolutePath, errors);
     }
+    await validateWorkspaceConfig(
+      evalCase.workspace,
+      absolutePath,
+      errors,
+      `${location}.workspace`
+    );
   }
-  if (isObject(parsed.workspace)) {
-    validateWorkspaceRepoConfig(parsed.workspace, absolutePath, errors);
-  }
+  await validateWorkspaceConfig(parsed.workspace, absolutePath, errors, "workspace");
   return {
     valid: errors.filter((e) => e.severity === "error").length === 0,
     filePath: absolutePath,
@@ -2367,6 +2398,41 @@ async function validateEvalFile(filePath) {
     errors
   };
 }
+async function validateWorkspaceConfig(workspace, evalFilePath, errors, location) {
+  if (workspace === void 0) {
+    return;
+  }
+  if (isObject(workspace)) {
+    validateWorkspaceRepoConfig(workspace, evalFilePath, errors);
+    return;
+  }
+  if (typeof workspace !== "string") {
+    return;
+  }
+  const workspacePath = path22.resolve(path22.dirname(evalFilePath), workspace);
+  try {
+    const workspaceContent = await readFile22(workspacePath, "utf8");
+    const parsedWorkspace = interpolateEnv(parse2(workspaceContent), process.env);
+    if (!isObject(parsedWorkspace)) {
+      errors.push({
+        severity: "error",
+        filePath: evalFilePath,
+        location,
+        message: `External workspace file must contain a YAML object: ${workspace}`
+      });
+      return;
+    }
+    validateWorkspaceRepoConfig(parsedWorkspace, workspacePath, errors);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    errors.push({
+      severity: "error",
+      filePath: evalFilePath,
+      location,
+      message: `Failed to load external workspace file '${workspace}': ${message}`
+    });
+  }
+}
 function validateWorkspaceRepoConfig(workspace, filePath, errors) {
   const repos = workspace.repos;
   const hooks = workspace.hooks;
@@ -2375,8 +2441,21 @@ function validateWorkspaceRepoConfig(workspace, filePath, errors) {
   if (Array.isArray(repos)) {
     for (const repo of repos) {
       if (!isObject(repo)) continue;
+      const source = repo.source;
       const checkout = repo.checkout;
       const clone = repo.clone;
+      if (isObject(source) && isObject(checkout)) {
+        const sourceType = source.type;
+        const resolve = checkout.resolve;
+        if (sourceType === "local" && typeof resolve === "string") {
+          errors.push({
+            severity: "warning",
+            filePath,
+            location: `workspace.repos[path=${repo.path}]`,
+            message: "checkout.resolve has no effect for a local source. Use source.type to choose where the repo comes from; keep checkout.ref or checkout.ancestor only when pinning a local source."
+          });
+        }
+      }
       if (isObject(checkout) && isObject(clone)) {
         const ancestor = checkout.ancestor;
         const depth = clone.depth;
@@ -3141,31 +3220,6 @@ async function validateConfigFile(filePath) {
       return { valid: false, filePath, fileType: "config", errors };
     }
     const config = parsed;
-    const guidelinePatterns = config.guideline_patterns;
-    if (guidelinePatterns !== void 0) {
-      if (!Array.isArray(guidelinePatterns)) {
-        errors.push({
-          severity: "error",
-          filePath,
-          location: "guideline_patterns",
-          message: "Field 'guideline_patterns' must be an array"
-        });
-      } else if (!guidelinePatterns.every((p) => typeof p === "string")) {
-        errors.push({
-          severity: "error",
-          filePath,
-          location: "guideline_patterns",
-          message: "All entries in 'guideline_patterns' must be strings"
-        });
-      } else if (guidelinePatterns.length === 0) {
-        errors.push({
-          severity: "warning",
-          filePath,
-          location: "guideline_patterns",
-          message: "Field 'guideline_patterns' is empty. Consider removing it or adding patterns."
-        });
-      }
-    }
     const evalPatterns = config.eval_patterns;
     if (evalPatterns !== void 0) {
       if (!Array.isArray(evalPatterns)) {
@@ -3202,13 +3256,7 @@ async function validateConfigFile(filePath) {
         });
       }
     }
-    const allowedFields = /* @__PURE__ */ new Set([
-      "$schema",
-      "guideline_patterns",
-      "eval_patterns",
-      "required_version",
-      "execution"
-    ]);
+    const allowedFields = /* @__PURE__ */ new Set(["$schema", "eval_patterns", "required_version", "execution"]);
     const unexpectedFields = Object.keys(config).filter((key) => !allowedFields.has(key));
     if (unexpectedFields.length > 0) {
       errors.push({
@@ -3614,6 +3662,34 @@ function normalizeOptionalNumber(value) {
 function normalizeWorkspaceMode(value) {
   return value === "pooled" || value === "temp" || value === "static" ? value : void 0;
 }
+function normalizeOutputMessages(cliValue) {
+  if (cliValue === void 0) {
+    return 1;
+  }
+  if (cliValue === "all") {
+    return "all";
+  }
+  const parsed = Number.parseInt(cliValue, 10);
+  if (Number.isNaN(parsed) || !Number.isInteger(parsed) || parsed < 1) {
+    console.warn(
+      `Warning: Invalid --output-messages value '${cliValue}'. Must be a positive integer or 'all'. Defaulting to 1.`
+    );
+    return 1;
+  }
+  return parsed;
+}
+function trimOutputMessages(output, outputMessages) {
+  const messages = output ?? [];
+  if (outputMessages === "all") {
+    return messages.map((m) => ({ role: m.role, content: m.content }));
+  }
+  if (outputMessages === 1) {
+    const lastAssistant = messages.filter((m) => m.role === "assistant").at(-1);
+    return lastAssistant ? [{ role: lastAssistant.role, content: lastAssistant.content }] : [];
+  }
+  const sliced = messages.slice(-outputMessages);
+  return sliced.map((m) => ({ role: m.role, content: m.content }));
+}
 function normalizeOptions(rawOptions, config, yamlExecution) {
   const cliFormat = normalizeString(rawOptions.outputFormat);
   const configFormat = config?.output?.format;
@@ -3693,7 +3769,8 @@ function normalizeOptions(rawOptions, config, yamlExecution) {
     benchmarkJson: normalizeString(rawOptions.benchmarkJson),
     artifacts: normalizeString(rawOptions.artifacts),
     graderTarget: normalizeString(rawOptions.graderTarget),
-    model: normalizeString(rawOptions.model)
+    model: normalizeString(rawOptions.model),
+    outputMessages: normalizeOutputMessages(normalizeString(rawOptions.outputMessages))
   };
 }
 async function ensureFileExists(filePath, description) {
@@ -3942,8 +4019,12 @@ async function runSingleEvalFile(params) {
     streamCallbacks: streamingObserver?.getStreamCallbacks(),
     onResult: async (result) => {
       streamingObserver?.finalizeEvalCase(result.score, result.error);
-      const { output: _, ...resultWithoutTrace } = result;
-      await outputWriter.append(resultWithoutTrace);
+      const trimmedOutput = trimOutputMessages(result.output, options.outputMessages);
+      const trimmedResult = {
+        ...result,
+        output: trimmedOutput
+      };
+      await outputWriter.append(trimmedResult);
       if (otelExporter && !streamingObserver) {
         try {
           await otelExporter.exportResult(result);
@@ -4044,7 +4125,7 @@ async function runEvalCommand(input) {
   const useFileExport = !!(options.otelFile || options.traceFile);
   if (options.exportOtel || useFileExport) {
     try {
-      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-VP6AXX6B.js");
+      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-PIOSPBKX.js");
       let endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
       let headers = {};
       if (options.otelBackend) {
@@ -4386,4 +4467,4 @@ export {
   selectTarget,
   runEvalCommand
 };
-//# sourceMappingURL=chunk-7YS6YNJZ.js.map
+//# sourceMappingURL=chunk-GC5P5HHZ.js.map