agentv 4.35.1 → 4.37.0-next.1

package/dist/{chunk-KJGYL3M3.js → chunk-N6E5XFOM.js}

@@ -1,41 +1,20 @@
 import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
 import {
-  toTranscriptJsonLines
-} from "./chunk-KNF3AGCI.js";
+  traceToTranscriptJsonLines
+} from "./chunk-OYI35QFW.js";
 import {
   DEFAULT_THRESHOLD,
-  parseYamlValue
-} from "./chunk-6QEIZ33V.js";
+  buildTraceEnvelopeFromEvaluationResult,
+  buildTraceFromMessages,
+  extractLastAssistantContent,
+  parseYamlValue,
+  toTraceEnvelopeWire
+} from "./chunk-RL4S2FBZ.js";
 
 // src/commands/eval/artifact-writer.ts
 import { mkdir as mkdir2, readFile as readFile2, writeFile as writeFile2 } from "node:fs/promises";
 import path3 from "node:path";
 
-// src/utils/case-conversion.ts
-function toSnakeCase(str) {
-  if (/^[A-Z]/.test(str)) {
-    return str;
-  }
-  return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
-}
-function toSnakeCaseDeep(obj) {
-  if (obj === null || obj === void 0) {
-    return obj;
-  }
-  if (Array.isArray(obj)) {
-    return obj.map((item) => toSnakeCaseDeep(item));
-  }
-  if (typeof obj === "object") {
-    const result = {};
-    for (const [key, value] of Object.entries(obj)) {
-      const snakeKey = toSnakeCase(key);
-      result[snakeKey] = toSnakeCaseDeep(value);
-    }
-    return result;
-  }
-  return obj;
-}
-
 // src/commands/eval/result-layout.ts
 import { existsSync, statSync } from "node:fs";
 import path from "node:path";
@@ -114,6 +93,33 @@ import { createHash } from "node:crypto";
 import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
 import path2 from "node:path";
 import { stringify as stringifyYaml } from "yaml";
+
+// src/utils/case-conversion.ts
+function toSnakeCase(str) {
+  if (/^[A-Z]/.test(str)) {
+    return str;
+  }
+  return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
+}
+function toSnakeCaseDeep(obj) {
+  if (obj === null || obj === void 0) {
+    return obj;
+  }
+  if (Array.isArray(obj)) {
+    return obj.map((item) => toSnakeCaseDeep(item));
+  }
+  if (typeof obj === "object") {
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+      const snakeKey = toSnakeCase(key);
+      result[snakeKey] = toSnakeCaseDeep(value);
+    }
+    return result;
+  }
+  return obj;
+}
+
+// src/commands/eval/task-bundle.ts
 var TASK_DIRNAME = "task";
 var TASK_EVAL_FILENAME = "EVAL.yaml";
 var TASK_TARGETS_FILENAME = "targets.yaml";
@@ -528,18 +534,8 @@ function isExecutionError(result) {
   return result.executionStatus === "execution_error";
 }
 function countToolCalls(result) {
-  const toolCalls = {};
-  let total = 0;
-  const trace = result.trace;
-  if (trace?.steps) {
-    for (const step of trace.steps) {
-      if (step.toolName || step.type === "tool") {
-        const name = step.toolName ?? "unknown";
-        toolCalls[name] = (toolCalls[name] ?? 0) + 1;
-        total += 1;
-      }
-    }
-  }
+  const toolCalls = { ...result.trace?.toolCalls ?? {} };
+  const total = Object.values(toolCalls).reduce((sum, count) => sum + count, 0);
   return { toolCalls, total };
 }
 function parseWorkspaceChanges(fileChanges) {
@@ -588,6 +584,66 @@ function buildEvaluators(scores) {
     details: s.details
   }));
 }
+function toIndexAssertion(assertion) {
+  return {
+    text: assertion.text,
+    passed: assertion.passed,
+    evidence: assertion.evidence
+  };
+}
+function toIndexScore(score) {
+  return {
+    name: score.name,
+    type: score.type,
+    score: score.score,
+    weight: score.weight,
+    verdict: score.verdict,
+    assertions: score.assertions.map(toIndexAssertion),
+    raw_request: score.rawRequest,
+    input: score.input,
+    target: score.target,
+    scores: score.scores?.map(toIndexScore),
+    details: score.details,
+    token_usage: score.tokenUsage,
+    duration_ms: score.durationMs,
+    started_at: score.startedAt,
+    ended_at: score.endedAt
+  };
+}
+function toIndexScores(scores) {
+  return scores?.map(toIndexScore);
+}
+function dropUndefined(value) {
+  return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== void 0));
+}
+function toIndexRerunSource(value) {
+  if (!isRecord2(value)) {
+    return void 0;
+  }
+  return dropUndefined({
+    mode: value.mode,
+    source_run_dir: value.sourceRunDir,
+    source_index_path: value.sourceIndexPath,
+    source_artifact_dir: value.sourceArtifactDir,
+    source_task_dir: value.sourceTaskDir,
+    source_test_id: value.sourceTestId,
+    source_target: value.sourceTarget,
+    source_timestamp: value.sourceTimestamp
+  });
+}
+function toIndexMetadata(metadata) {
+  if (!metadata) {
+    return void 0;
+  }
+  const rerunSource = toIndexRerunSource(metadata.rerunSource);
+  if (!rerunSource) {
+    return { ...metadata };
+  }
+  return {
+    ...Object.fromEntries(Object.entries(metadata).filter(([key]) => key !== "rerunSource")),
+    rerun_source: rerunSource
+  };
+}
 function buildGradingArtifact(result) {
   const assertions = buildAssertions(result);
   const passed = assertions.filter((e) => e.passed).length;
@@ -611,7 +667,7 @@ function buildGradingArtifact(result) {
     graders: buildEvaluators(result.scores),
     workspace_changes: parseWorkspaceChanges(result.fileChanges),
     conversation: result.conversationId ? {
-      turns: result.trace ? result.trace.steps?.length ?? 0 : 0,
+      turns: result.trace?.messages.filter((message) => message.role === "assistant").length ?? 0,
       conversation_id: result.conversationId
     } : void 0
   };
@@ -823,6 +879,34 @@ function buildTaskBundleIndexFields(outputDir, taskBundle) {
     ...taskBundle.gradersPath ? { graders_path: toRelativeArtifactPath(outputDir, taskBundle.gradersPath) } : {}
   };
 }
+function findResultSourceTest(result, testByTestId) {
+  return testByTestId.get(result.testId ?? "unknown");
+}
+function resolveEnvelopeEvalPath(result, testByTestId, fallbackEvalFile) {
+  const source = findResultSourceTest(result, testByTestId)?.source;
+  return source?.evalFileRepoPath ?? source?.evalFilePath ?? fallbackEvalFile;
+}
+async function writeTraceEnvelopeSidecar(params) {
+  const hasTranscript = params.result.output.length > 0 || params.result.trace.messages.length > 0;
+  const envelope = buildTraceEnvelopeFromEvaluationResult(params.result, {
+    evalPath: params.evalPath,
+    runId: path3.basename(params.outputDir),
+    experiment: params.experiment,
+    source: { path: RESULT_INDEX_FILENAME },
+    artifacts: {
+      envelope_path: "outputs/trace-envelope.json",
+      answer_path: params.result.output.length > 0 ? "outputs/answer.md" : void 0,
+      response_path: params.result.output.length > 0 ? "outputs/response.md" : void 0,
+      transcript_path: hasTranscript ? "outputs/transcript.jsonl" : void 0
+    }
+  });
+  await writeFile2(
+    path3.join(params.outputsDir, "trace-envelope.json"),
+    `${JSON.stringify(toTraceEnvelopeWire(envelope), null, 2)}
+`,
+    "utf8"
+  );
+}
 function buildIndexArtifactEntry(result, options) {
   return {
     timestamp: result.timestamp,
@@ -832,7 +916,12 @@ function buildIndexArtifactEntry(result, options) {
     conversation_id: result.conversationId,
     score: result.score,
     target: result.target ?? "unknown",
-    scores: result.scores ? toSnakeCaseDeep(result.scores) : void 0,
+    token_usage: result.tokenUsage,
+    cost_usd: result.costUsd,
+    duration_ms: result.durationMs,
+    start_time: result.startTime,
+    end_time: result.endTime,
+    scores: toIndexScores(result.scores),
     execution_status: result.executionStatus,
     error: result.error,
     failure_stage: result.failureStage,
@@ -842,15 +931,19 @@ function buildIndexArtifactEntry(result, options) {
     grading_path: toRelativeArtifactPath(options.outputDir, options.gradingPath),
     timing_path: toRelativeArtifactPath(options.outputDir, options.timingPath),
     output_path: options.outputPath ? toRelativeArtifactPath(options.outputDir, options.outputPath) : void 0,
+    answer_path: options.answerPath ? toRelativeArtifactPath(options.outputDir, options.answerPath) : void 0,
+    transcript_path: options.transcriptPath ? toRelativeArtifactPath(options.outputDir, options.transcriptPath) : void 0,
     input_path: options.inputPath ? toRelativeArtifactPath(options.outputDir, options.inputPath) : void 0,
+    response_path: options.responsePath ? toRelativeArtifactPath(options.outputDir, options.responsePath) : void 0,
     ...buildTaskBundleIndexFields(options.outputDir, options.taskBundle),
-    metadata: result.metadata
+    metadata: toIndexMetadata(result.metadata)
   };
 }
 function buildResultIndexArtifact(result, taskBundle) {
   const artifactSubdir = buildArtifactSubdir(result);
   const input = extractInput(result);
-  const hasResponse = Array.isArray(result.output) && result.output.length > 0;
+  const hasAnswer = result.output.length > 0;
+  const hasTranscript = result.trace.messages.length > 0 || result.trace.events.length > 0;
   return {
     timestamp: result.timestamp,
     test_id: result.testId ?? "unknown",
@@ -859,7 +952,12 @@ function buildResultIndexArtifact(result, taskBundle) {
     conversation_id: result.conversationId,
     score: result.score,
     target: result.target ?? "unknown",
-    scores: result.scores ? toSnakeCaseDeep(result.scores) : void 0,
+    token_usage: result.tokenUsage,
+    cost_usd: result.costUsd,
+    duration_ms: result.durationMs,
+    start_time: result.startTime,
+    end_time: result.endTime,
+    scores: toIndexScores(result.scores),
     execution_status: result.executionStatus,
     error: result.error,
     failure_stage: result.failureStage,
@@ -869,8 +967,10 @@ function buildResultIndexArtifact(result, taskBundle) {
     grading_path: path3.posix.join(artifactSubdir, "grading.json"),
     timing_path: path3.posix.join(artifactSubdir, "timing.json"),
     input_path: input ? path3.posix.join(artifactSubdir, "input.md") : void 0,
-    output_path: hasResponse ? path3.posix.join(artifactSubdir, "outputs", "response.md") : void 0,
-    response_path: hasResponse ? path3.posix.join(artifactSubdir, "outputs", "response.md") : void 0,
+    output_path: hasAnswer ? path3.posix.join(artifactSubdir, "outputs", "answer.md") : void 0,
+    answer_path: hasAnswer ? path3.posix.join(artifactSubdir, "outputs", "answer.md") : void 0,
+    transcript_path: hasTranscript ? path3.posix.join(artifactSubdir, "outputs", "transcript.jsonl") : void 0,
+    response_path: hasAnswer ? path3.posix.join(artifactSubdir, "outputs", "response.md") : void 0,
     ...taskBundle ? {
       task_dir: path3.posix.join(artifactSubdir, "task"),
       eval_path: path3.posix.join(artifactSubdir, "task", "EVAL.yaml"),
@@ -878,14 +978,23 @@ function buildResultIndexArtifact(result, taskBundle) {
       ...taskBundle.filesPath ? { files_path: path3.posix.join(artifactSubdir, "task", "files") } : {},
       ...taskBundle.gradersPath ? { graders_path: path3.posix.join(artifactSubdir, "task", "graders") } : {}
     } : {},
-    metadata: result.metadata
+    metadata: toIndexMetadata(result.metadata)
   };
 }
 async function writeJsonlFile(filePath, records) {
-  const content = records.length === 0 ? "" : `${records.map((record) => JSON.stringify(toSnakeCaseDeep(record))).join("\n")}
+  const content = records.length === 0 ? "" : `${records.map((record) => JSON.stringify(record)).join("\n")}
 `;
   await writeFile2(filePath, content, "utf8");
 }
+async function writeTranscriptJsonl(filePath, result) {
+  const lines = traceToTranscriptJsonLines(result.trace, {
+    testId: result.testId,
+    target: result.target
+  });
+  const content = lines.length > 0 ? `${lines.map((line) => JSON.stringify(line)).join("\n")}
+` : "";
+  await writeFile2(filePath, content, "utf8");
+}
 function isRecord2(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
@@ -977,11 +1086,28 @@ function isOutputMessage(value) {
 function isExecutionStatus(value) {
   return typeof value === "string" && EXECUTION_STATUSES.has(value);
 }
+function isTraceRecord(value) {
+  return !!value && typeof value === "object" && !Array.isArray(value) && Array.isArray(value.messages) && Array.isArray(value.events);
+}
 function normalizeParsedResult(value) {
   if (!value || typeof value !== "object" || Array.isArray(value)) {
     return void 0;
   }
   const result = value;
+  const legacyOutputMessages = Array.isArray(result.output) ? result.output.filter(isOutputMessage) : void 0;
+  const output = typeof result.output === "string" ? result.output : extractLastAssistantContent(legacyOutputMessages);
+  const legacySummary = result.trace && typeof result.trace === "object" && !Array.isArray(result.trace) ? result.trace : void 0;
+  const trace = isTraceRecord(result.trace) ? result.trace : buildTraceFromMessages({
+    input: Array.isArray(result.input) ? result.input : [],
+    output: legacyOutputMessages,
+    summary: legacySummary,
+    finalOutput: output,
+    tokenUsage: result.tokenUsage,
+    costUsd: typeof result.costUsd === "number" ? result.costUsd : void 0,
+    durationMs: typeof result.durationMs === "number" ? result.durationMs : void 0,
+    target: typeof result.target === "string" ? result.target : void 0,
+    testId: typeof result.testId === "string" ? result.testId : void 0
+  });
   return {
     ...result,
     timestamp: typeof result.timestamp === "string" ? result.timestamp : (/* @__PURE__ */ new Date(0)).toISOString(),
@@ -989,7 +1115,8 @@ function normalizeParsedResult(value) {
     score: typeof result.score === "number" ? result.score : 0,
     assertions: Array.isArray(result.assertions) ? result.assertions.filter(isAssertionEntry) : [],
     target: typeof result.target === "string" ? result.target : "unknown",
-    output: Array.isArray(result.output) ? result.output.filter(isOutputMessage) : [],
+    output,
+    trace,
     executionStatus: isExecutionStatus(result.executionStatus) ? result.executionStatus : "ok"
   };
 }
@@ -1021,23 +1148,10 @@ async function writeArtifacts(jsonlPath, outputDir, options) {
 function buildTranscriptMessageLines(results) {
   const lines = [];
   for (const result of results) {
-    const transcriptLines = toTranscriptJsonLines(
-      {
-        messages: [...result.input ?? [], ...result.output],
-        source: {
-          provider: result.target,
-          sessionId: result.conversationId ?? result.testId,
-          startedAt: result.timestamp
-        },
-        tokenUsage: result.tokenUsage,
-        durationMs: result.durationMs,
-        costUsd: result.costUsd
-      },
-      {
-        testId: result.testId,
-        target: result.target
-      }
-    );
+    const transcriptLines = traceToTranscriptJsonLines(result.trace, {
+      testId: result.testId,
+      target: result.target
+    });
     lines.push(...transcriptLines.map((line) => JSON.stringify(line)));
   }
   return lines.length > 0 ? `${lines.join("\n")}
@@ -1115,15 +1229,22 @@ async function writePerTestArtifacts(results, outputDir, options) {
     if (input) {
       await writeFile2(path3.join(testDir, "input.md"), input, "utf8");
     }
-    if (result.output && result.output.length > 0) {
-      const outputsDir = path3.join(testDir, "outputs");
-      await mkdir2(outputsDir, { recursive: true });
-      await writeFile2(
-        path3.join(outputsDir, "response.md"),
-        formatOutputMarkdown(result.output),
-        "utf8"
-      );
+    const outputsDir = path3.join(testDir, "outputs");
+    await mkdir2(outputsDir, { recursive: true });
+    if (result.output.length > 0) {
+      await writeFile2(path3.join(outputsDir, "answer.md"), result.output, "utf8");
+      await writeFile2(path3.join(outputsDir, "response.md"), result.output, "utf8");
     }
+    if (result.output.length > 0 || result.trace.messages.length > 0) {
+      await writeTranscriptJsonl(path3.join(outputsDir, "transcript.jsonl"), result);
+    }
+    await writeTraceEnvelopeSidecar({
+      result,
+      outputDir,
+      outputsDir,
+      evalPath: resolveEnvelopeEvalPath(result, testByTestId),
+      experiment: options?.experiment
+    });
     const taskBundle = await materializeTaskBundleForResult({
       result,
       testDir,
@@ -1164,15 +1285,22 @@ async function writeArtifactsFromResults(results, outputDir, options) {
     if (input) {
       await writeFile2(path3.join(testDir, "input.md"), input, "utf8");
     }
-    if (result.output && result.output.length > 0) {
-      const outputsDir = path3.join(testDir, "outputs");
-      await mkdir2(outputsDir, { recursive: true });
-      await writeFile2(
-        path3.join(outputsDir, "response.md"),
-        formatOutputMarkdown(result.output),
-        "utf8"
-      );
+    const outputsDir = path3.join(testDir, "outputs");
+    await mkdir2(outputsDir, { recursive: true });
+    if (result.output.length > 0) {
+      await writeFile2(path3.join(outputsDir, "answer.md"), result.output, "utf8");
+      await writeFile2(path3.join(outputsDir, "response.md"), result.output, "utf8");
+    }
+    if (result.output.length > 0 || result.trace.messages.length > 0) {
+      await writeTranscriptJsonl(path3.join(outputsDir, "transcript.jsonl"), result);
     }
+    await writeTraceEnvelopeSidecar({
+      result,
+      outputDir,
+      outputsDir,
+      evalPath: resolveEnvelopeEvalPath(result, testByTestId, options?.evalFile),
+      experiment: options?.experiment
+    });
     const taskBundle = await materializeTaskBundleForResult({
       result,
       testDir,
@@ -1231,4 +1359,4 @@ export {
   writePerTestArtifacts,
   writeArtifactsFromResults
 };
-//# sourceMappingURL=chunk-KJGYL3M3.js.map
+//# sourceMappingURL=chunk-N6E5XFOM.js.map