agentv 4.14.0-next.1 → 4.15.0-next.1

package/dist/{chunk-YFQMID6V.js → chunk-CJFA423D.js}

@@ -41,12 +41,12 @@ import {
   subscribeToPiLogEntries,
   syncResultsRepo,
   toCamelCaseDeep
-} from "./chunk-HP3ZUKYB.js";
+} from "./chunk-VSWJTL3T.js";
 
 // package.json
 var package_default = {
   name: "agentv",
-  version: "4.14.0-next.1",
+  version: "4.15.0-next.1",
   description: "CLI entry point for AgentV",
   type: "module",
   repository: {
@@ -1491,6 +1491,37 @@ async function writeArtifactsFromResults(results, outputDir, options) {
   await writeFile(benchmarkPath, `${JSON.stringify(benchmark, null, 2)}
 `, "utf8");
   await writeJsonlFile(indexPath, indexRecords);
+  const transcriptPath = path7.join(outputDir, "transcript.jsonl");
+  const transcriptLines = results.map((result) => {
+    let inputText = "";
+    if (typeof result.input === "string") {
+      inputText = result.input;
+    } else if (Array.isArray(result.input)) {
+      const firstUserMsg = result.input.find((m) => m.role === "user");
+      inputText = typeof firstUserMsg?.content === "string" ? firstUserMsg.content : "";
+    }
+    return {
+      input: inputText,
+      output: result.output,
+      token_usage: result.tokenUsage ? {
+        input: result.tokenUsage.input,
+        output: result.tokenUsage.output,
+        cached: result.tokenUsage.cached
+      } : void 0,
+      duration_ms: result.durationMs,
+      cost_usd: result.costUsd,
+      source: {
+        provider: result.target,
+        session_id: result.conversationId ?? result.testId,
+        timestamp: result.timestamp
+      }
+    };
+  });
+  await writeFile(
+    transcriptPath,
+    transcriptLines.map((line) => JSON.stringify(line)).join("\n") + (transcriptLines.length ? "\n" : ""),
+    "utf8"
+  );
   return { testArtifactDir, timingPath, benchmarkPath, indexPath };
 }
 
@@ -3147,7 +3178,14 @@ var KNOWN_TEST_FIELDS = /* @__PURE__ */ new Set([
   "metadata",
   "conversation_id",
   "suite",
-  "note"
+  "note",
+  "depends_on",
+  "on_dependency_failure",
+  "mode",
+  "turns",
+  "aggregation",
+  "on_turn_failure",
+  "window_size"
 ]);
 var NAME_PATTERN = /^[a-z0-9-]+$/;
 function isObject(value) {
@@ -3360,6 +3398,7 @@ async function validateEvalFile(filePath) {
     if (assertField !== void 0) {
       validateAssertArray(assertField, location, absolutePath, errors);
     }
+    validateConversationMode(evalCase, location, absolutePath, errors);
     await validateWorkspaceConfig(
       evalCase.workspace,
       absolutePath,
@@ -3456,16 +3495,6 @@ function validateWorkspaceRepoConfig(workspace, filePath, errors) {
       }
     }
   }
-  if (isObject(afterEachHook) && afterEachHook.reset && afterEachHook.reset !== "none") {
-    if (!Array.isArray(repos) || repos.length === 0) {
-      errors.push({
-        severity: "warning",
-        filePath,
-        location: "workspace.hooks.after_each",
-        message: `hooks.after_each.reset '${afterEachHook.reset}' has no effect without repos.`
-      });
-    }
-  }
   if (isObject(afterEachHook) && afterEachHook.reset && isolation === "per_test") {
     errors.push({
       severity: "warning",
@@ -3712,6 +3741,86 @@ function validateContentForRoleMarkers(content, location, filePath, errors) {
     }
   }
 }
+function validateConversationMode(evalCase, location, filePath, errors) {
+  const mode = evalCase.mode;
+  const turns = evalCase.turns;
+  const aggregation = evalCase.aggregation;
+  const onTurnFailure = evalCase.on_turn_failure;
+  const windowSize = evalCase.window_size;
+  const isConversationMode = mode === "conversation";
+  if (turns !== void 0 && !isConversationMode) {
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.turns`,
+      message: "'turns' requires mode: conversation"
+    });
+  }
+  if (isConversationMode && (!Array.isArray(turns) || turns.length === 0)) {
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.mode`,
+      message: "mode: conversation requires a non-empty 'turns' array"
+    });
+  }
+  if (isConversationMode && Array.isArray(turns) && evalCase.expected_output !== void 0) {
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.expected_output`,
+      message: "Top-level 'expected_output' is not allowed with mode: conversation (use per-turn expected_output instead)"
+    });
+  }
+  if (aggregation !== void 0 && !isConversationMode) {
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.aggregation`,
+      message: "'aggregation' requires mode: conversation"
+    });
+  }
+  if (onTurnFailure !== void 0 && !isConversationMode) {
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.on_turn_failure`,
+      message: "'on_turn_failure' requires mode: conversation"
+    });
+  }
+  if (windowSize !== void 0 && !isConversationMode) {
+    errors.push({
+      severity: "error",
+      filePath,
+      location: `${location}.window_size`,
+      message: "'window_size' requires mode: conversation"
+    });
+  }
+  if (isConversationMode && Array.isArray(turns)) {
+    for (let i = 0; i < turns.length; i++) {
+      const turn = turns[i];
+      if (!isObject(turn)) {
+        errors.push({
+          severity: "error",
+          filePath,
+          location: `${location}.turns[${i}]`,
+          message: "Turn must be an object"
+        });
+        continue;
+      }
+      const turnInput = turn.input;
+      const isEmpty = turnInput === void 0 || turnInput === "" || typeof turnInput === "string" && turnInput.trim() === "" || Array.isArray(turnInput) && turnInput.length === 0;
+      if (isEmpty) {
+        errors.push({
+          severity: "error",
+          filePath,
+          location: `${location}.turns[${i}].input`,
+          message: "Each turn must have a non-empty input"
+        });
+      }
+    }
+  }
+}
 function isObject2(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
@@ -5242,7 +5351,7 @@ async function runEvalCommand(input) {
   const useFileExport = !!options.otelFile;
   if (options.exportOtel || useFileExport) {
     try {
-      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-Y3NRIBUT.js");
+      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-G6XAYD4R.js");
       let endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
       let headers = {};
       if (options.otelBackend) {
@@ -5438,7 +5547,7 @@ async function runEvalCommand(input) {
   const activeTestFiles = resolvedTestFiles.filter((f) => fileMetadata.has(f));
   let transcriptProviderFactory;
   if (options.transcript) {
-    const { TranscriptProvider } = await import("./dist-Y3NRIBUT.js");
+    const { TranscriptProvider } = await import("./dist-G6XAYD4R.js");
     const transcriptProvider = await TranscriptProvider.fromFile(options.transcript);
     const totalTests = [...fileMetadata.values()].reduce(
       (sum, meta) => sum + meta.testCases.length,
@@ -5760,4 +5869,4 @@ export {
   getCategories,
   filterByCategory
 };
-//# sourceMappingURL=chunk-YFQMID6V.js.map
+//# sourceMappingURL=chunk-CJFA423D.js.map