agentv 3.11.0 → 3.12.0

package/dist/{chunk-JEW3FEO7.js → chunk-VLOFRXH4.js}

@@ -27,12 +27,12 @@ import {
   subscribeToCopilotCliLogEntries,
   subscribeToCopilotSdkLogEntries,
   subscribeToPiLogEntries
-} from "./chunk-EZGWZVVK.js";
+} from "./chunk-UYBLUYHN.js";
 
 // package.json
 var package_default = {
   name: "agentv",
-  version: "3.11.0",
+  version: "3.12.0",
   description: "CLI entry point for AgentV",
   type: "module",
   repository: {
@@ -65,8 +65,6 @@ var package_default = {
     "@github/copilot-sdk": "^0.1.25",
     "@hono/node-server": "^1.19.11",
     "@inquirer/prompts": "^8.2.1",
-    "@mariozechner/pi-agent-core": "^0.54.2",
-    "@mariozechner/pi-ai": "^0.54.2",
     "@openai/codex-sdk": "^0.104.0",
     "cmd-ts": "^0.14.3",
     dotenv: "^16.4.5",
@@ -77,6 +75,14 @@ var package_default = {
     semver: "^7.7.4",
     yaml: "^2.6.1"
   },
+  peerDependencies: {
+    "@mariozechner/pi-coding-agent": "^0.62.0"
+  },
+  peerDependenciesMeta: {
+    "@mariozechner/pi-coding-agent": {
+      optional: true
+    }
+  },
   devDependencies: {
     "@agentv/core": "workspace:*",
     "@types/semver": "^7.7.1",
@@ -206,7 +212,7 @@ async function discoverTargetsFile(options) {
 // src/commands/eval/run-eval.ts
 import { constants as constants4, mkdirSync } from "node:fs";
 import { access as access4 } from "node:fs/promises";
-import path13 from "node:path";
+import path15 from "node:path";
 import { pathToFileURL } from "node:url";
 
 // src/version-check.ts
@@ -265,7 +271,82 @@ async function promptContinue() {
 
 // src/commands/eval/artifact-writer.ts
 import { mkdir, readFile, writeFile } from "node:fs/promises";
+import path4 from "node:path";
+
+// src/utils/case-conversion.ts
+function toSnakeCase(str) {
+  if (/^[A-Z]/.test(str)) {
+    return str;
+  }
+  return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
+}
+function toSnakeCaseDeep(obj) {
+  if (obj === null || obj === void 0) {
+    return obj;
+  }
+  if (Array.isArray(obj)) {
+    return obj.map((item) => toSnakeCaseDeep(item));
+  }
+  if (typeof obj === "object") {
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+      const snakeKey = toSnakeCase(key);
+      result[snakeKey] = toSnakeCaseDeep(value);
+    }
+    return result;
+  }
+  return obj;
+}
+
+// src/commands/eval/result-layout.ts
+import { existsSync, statSync } from "node:fs";
 import path3 from "node:path";
+var RESULT_INDEX_FILENAME = "index.jsonl";
+var LEGACY_RESULTS_FILENAME = "results.jsonl";
+function createRunDirName(timestamp = /* @__PURE__ */ new Date()) {
+  return `eval_${timestamp.toISOString().replace(/[:.]/g, "-")}`;
+}
+function buildDefaultRunDir(cwd) {
+  return path3.join(cwd, ".agentv", "results", "raw", createRunDirName());
+}
+function resolveRunIndexPath(runDir) {
+  return path3.join(runDir, RESULT_INDEX_FILENAME);
+}
+function resolveRunLegacyResultsPath(runDir) {
+  return path3.join(runDir, LEGACY_RESULTS_FILENAME);
+}
+function resolveExistingRunPrimaryPath(runDir) {
+  const indexPath = resolveRunIndexPath(runDir);
+  if (existsSync(indexPath)) {
+    return indexPath;
+  }
+  const legacyPath = resolveRunLegacyResultsPath(runDir);
+  if (existsSync(legacyPath)) {
+    return legacyPath;
+  }
+  return void 0;
+}
+function isDirectoryPath(filePath) {
+  try {
+    return statSync(filePath).isDirectory();
+  } catch {
+    return false;
+  }
+}
+function resolveWorkspaceOrFilePath(filePath) {
+  if (!isDirectoryPath(filePath)) {
+    return filePath;
+  }
+  const existing = resolveExistingRunPrimaryPath(filePath);
+  if (!existing) {
+    throw new Error(
+      `Result workspace is missing ${RESULT_INDEX_FILENAME} and ${LEGACY_RESULTS_FILENAME}: ${filePath}`
+    );
+  }
+  return existing;
+}
+
+// src/commands/eval/artifact-writer.ts
 var PASS_THRESHOLD = 0.8;
 function computeStats(values) {
   if (values.length === 0) {
@@ -480,33 +561,74 @@ function buildBenchmarkArtifact(results, evalFile = "") {
     notes
   };
 }
-function buildAggregateGradingArtifact(results) {
-  const assertions = [];
-  for (const result of results) {
-    if (!result.assertions) continue;
-    const testId = result.testId ?? "unknown";
-    for (const a of result.assertions) {
-      assertions.push({
-        test_id: testId,
-        text: a.text,
-        passed: a.passed,
-        evidence: a.evidence ?? ""
-      });
-    }
+function safeArtifactPathSegment(value, fallback) {
+  const trimmed = value?.trim();
+  if (!trimmed) {
+    return fallback;
   }
-  const passed = assertions.filter((a) => a.passed).length;
-  const failed = assertions.filter((a) => !a.passed).length;
-  const total = assertions.length;
+  return trimmed.replace(/[/\\:*?"<>|]/g, "_");
+}
+function safeTestId(testId) {
+  return safeArtifactPathSegment(testId, "unknown");
+}
+function safeTargetId(target) {
+  return safeArtifactPathSegment(target, "default");
+}
+function getEvalSet(result) {
+  const record = result;
+  return result.eval_set ?? record.evalSet;
+}
+function buildArtifactSubdir(result) {
+  const segments = [];
+  const evalSet = getEvalSet(result);
+  if (evalSet) {
+    segments.push(safeArtifactPathSegment(evalSet, "default"));
+  }
+  segments.push(safeTestId(result.testId), safeTargetId(result.target));
+  return path4.posix.join(...segments);
+}
+function formatOutputMarkdown(output) {
+  return output.map((msg) => `@[${msg.role}]:
+${String(msg.content ?? "")}`).join("\n\n");
+}
+function extractInput(result) {
+  const input = result.input;
+  if (!input) return null;
+  if (typeof input === "string") return input;
+  if (Array.isArray(input) && input.length > 0) {
+    return formatOutputMarkdown(input);
+  }
+  return null;
+}
+function buildResultIndexArtifact(result) {
+  const artifactSubdir = buildArtifactSubdir(result);
+  const input = extractInput(result);
+  const hasResponse = Array.isArray(result.output) && result.output.length > 0;
   return {
-    assertions,
-    summary: {
-      passed,
-      failed,
-      total,
-      pass_rate: total > 0 ? Math.round(passed / total * 1e3) / 1e3 : 0
-    }
+    timestamp: result.timestamp,
+    test_id: result.testId ?? "unknown",
+    eval_set: getEvalSet(result),
+    conversation_id: result.conversationId,
+    score: result.score,
+    target: result.target ?? "unknown",
+    scores: result.scores ? toSnakeCaseDeep(result.scores) : void 0,
+    execution_status: result.executionStatus,
+    error: result.error,
+    failure_stage: result.failureStage,
+    failure_reason_code: result.failureReasonCode,
+    workspace_path: result.workspacePath,
+    grading_path: path4.posix.join(artifactSubdir, "grading.json"),
+    timing_path: path4.posix.join(artifactSubdir, "timing.json"),
+    input_path: input ? path4.posix.join(artifactSubdir, "input.md") : void 0,
+    output_path: hasResponse ? path4.posix.join(artifactSubdir, "outputs", "response.md") : void 0,
+    response_path: hasResponse ? path4.posix.join(artifactSubdir, "outputs", "response.md") : void 0
   };
 }
+async function writeJsonlFile(filePath, records) {
+  const content = records.length === 0 ? "" : `${records.map((record) => JSON.stringify(toSnakeCaseDeep(record))).join("\n")}
+`;
+  await writeFile(filePath, content, "utf8");
+}
 function toCamelCase(str) {
   return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase());
 }
@@ -544,17 +666,39 @@ function parseJsonlResults(content) {
   return results;
 }
 async function writeArtifactsFromResults(results, outputDir, options) {
-  const gradingDir = path3.join(outputDir, "grading");
-  const timingPath = path3.join(outputDir, "timing.json");
-  const benchmarkPath = path3.join(outputDir, "benchmark.json");
-  const aggregateGradingPath = path3.join(outputDir, "grading.json");
-  await mkdir(gradingDir, { recursive: true });
+  const testArtifactDir = outputDir;
+  const timingPath = path4.join(outputDir, "timing.json");
+  const benchmarkPath = path4.join(outputDir, "benchmark.json");
+  const indexPath = path4.join(outputDir, RESULT_INDEX_FILENAME);
+  const legacyResultsPath = options?.writeLegacyResults ? path4.join(outputDir, LEGACY_RESULTS_FILENAME) : void 0;
+  await mkdir(outputDir, { recursive: true });
+  const indexRecords = [];
   for (const result of results) {
     const grading = buildGradingArtifact(result);
-    const safeTestId = (result.testId ?? "unknown").replace(/[/\\:*?"<>|]/g, "_");
-    const gradingPath = path3.join(gradingDir, `${safeTestId}.json`);
+    const timing2 = buildTimingArtifact([result]);
+    const artifactSubdir = buildArtifactSubdir(result);
+    const testDir = path4.join(outputDir, artifactSubdir);
+    const gradingPath = path4.join(testDir, "grading.json");
+    const perTestTimingPath = path4.join(testDir, "timing.json");
+    await mkdir(testDir, { recursive: true });
     await writeFile(gradingPath, `${JSON.stringify(grading, null, 2)}
 `, "utf8");
+    await writeFile(perTestTimingPath, `${JSON.stringify(timing2, null, 2)}
+`, "utf8");
+    const input = extractInput(result);
+    if (input) {
+      await writeFile(path4.join(testDir, "input.md"), input, "utf8");
+    }
+    if (result.output && result.output.length > 0) {
+      const outputsDir = path4.join(testDir, "outputs");
+      await mkdir(outputsDir, { recursive: true });
+      await writeFile(
+        path4.join(outputsDir, "response.md"),
+        formatOutputMarkdown(result.output),
+        "utf8"
+      );
+    }
+    indexRecords.push(buildResultIndexArtifact(result));
   }
   const timing = buildTimingArtifact(results);
   await writeFile(timingPath, `${JSON.stringify(timing, null, 2)}
@@ -562,10 +706,11 @@ async function writeArtifactsFromResults(results, outputDir, options) {
   const benchmark = buildBenchmarkArtifact(results, options?.evalFile);
   await writeFile(benchmarkPath, `${JSON.stringify(benchmark, null, 2)}
 `, "utf8");
-  const aggregateGrading = buildAggregateGradingArtifact(results);
-  await writeFile(aggregateGradingPath, `${JSON.stringify(aggregateGrading, null, 2)}
-`, "utf8");
-  return { gradingDir, timingPath, benchmarkPath, aggregateGradingPath };
+  await writeJsonlFile(indexPath, indexRecords);
+  if (legacyResultsPath) {
+    await writeJsonlFile(legacyResultsPath, results);
+  }
+  return { testArtifactDir, timingPath, benchmarkPath, indexPath, legacyResultsPath };
 }
 
 // src/commands/eval/benchmark-writer.ts
@@ -616,13 +761,13 @@ async function writeBenchmarkJson(outputPath, results) {
 // src/commands/eval/env.ts
 import { constants as constants3 } from "node:fs";
 import { access as access3 } from "node:fs/promises";
-import path4 from "node:path";
+import path5 from "node:path";
 import { config as loadDotenv } from "dotenv";
 function uniqueDirs(directories) {
   const seen = /* @__PURE__ */ new Set();
   const result = [];
   for (const dir of directories) {
-    const absolute = path4.resolve(dir);
+    const absolute = path5.resolve(dir);
     if (seen.has(absolute)) {
       continue;
     }
@@ -641,14 +786,14 @@ async function fileExists2(filePath) {
 }
 function collectAncestorDirectories(start, boundary) {
   const directories = [];
-  const boundaryDir = path4.resolve(boundary);
-  let current = path4.resolve(start);
+  const boundaryDir = path5.resolve(boundary);
+  let current = path5.resolve(start);
   while (current !== void 0) {
     directories.push(current);
     if (current === boundaryDir) {
       break;
     }
-    const parent = path4.dirname(current);
+    const parent = path5.dirname(current);
     if (parent === current) {
       break;
     }
@@ -658,12 +803,12 @@ function collectAncestorDirectories(start, boundary) {
 }
 async function loadEnvFromHierarchy(options) {
   const { testFilePath, repoRoot, verbose } = options;
-  const testDir = path4.dirname(path4.resolve(testFilePath));
+  const testDir = path5.dirname(path5.resolve(testFilePath));
   const cwd = process.cwd();
   const searchDirs = uniqueDirs([...collectAncestorDirectories(testDir, repoRoot), repoRoot, cwd]);
   const envFiles = [];
   for (const dir of searchDirs) {
-    const candidate = path4.join(dir, ".env");
+    const candidate = path5.join(dir, ".env");
     if (await fileExists2(candidate)) {
       envFiles.push(candidate);
     }
@@ -685,11 +830,11 @@ async function loadEnvFromHierarchy(options) {
 }
 
 // src/commands/eval/output-writer.ts
-import path10 from "node:path";
+import path11 from "node:path";
 
 // src/commands/eval/html-writer.ts
 import { mkdir as mkdir2, writeFile as writeFile3 } from "node:fs/promises";
-import path5 from "node:path";
+import path6 from "node:path";
 
 // ../../node_modules/.bun/async-mutex@0.5.0/node_modules/async-mutex/index.mjs
 var E_TIMEOUT = new Error("timeout while waiting for mutex to become available");
@@ -908,7 +1053,7 @@ var HtmlWriter = class _HtmlWriter {
     this.filePath = filePath;
   }
   static async open(filePath) {
-    await mkdir2(path5.dirname(filePath), { recursive: true });
+    await mkdir2(path6.dirname(filePath), { recursive: true });
     const writer = new _HtmlWriter(filePath);
     await writer.writeHtml();
     return writer;
@@ -1419,34 +1564,7 @@ var SCRIPT = `
 
 // src/commands/eval/json-writer.ts
 import { mkdir as mkdir3, writeFile as writeFile4 } from "node:fs/promises";
-import path6 from "node:path";
-
-// src/utils/case-conversion.ts
-function toSnakeCase(str) {
-  if (/^[A-Z]/.test(str)) {
-    return str;
-  }
-  return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
-}
-function toSnakeCaseDeep(obj) {
-  if (obj === null || obj === void 0) {
-    return obj;
-  }
-  if (Array.isArray(obj)) {
-    return obj.map((item) => toSnakeCaseDeep(item));
-  }
-  if (typeof obj === "object") {
-    const result = {};
-    for (const [key, value] of Object.entries(obj)) {
-      const snakeKey = toSnakeCase(key);
-      result[snakeKey] = toSnakeCaseDeep(value);
-    }
-    return result;
-  }
-  return obj;
-}
-
-// src/commands/eval/json-writer.ts
+import path7 from "node:path";
 var JsonWriter = class _JsonWriter {
   filePath;
   results = [];
@@ -1455,7 +1573,7 @@ var JsonWriter = class _JsonWriter {
     this.filePath = filePath;
   }
   static async open(filePath) {
-    await mkdir3(path6.dirname(filePath), { recursive: true });
+    await mkdir3(path7.dirname(filePath), { recursive: true });
     return new _JsonWriter(filePath);
   }
   async append(result) {
@@ -1490,7 +1608,7 @@ var JsonWriter = class _JsonWriter {
 // src/commands/eval/jsonl-writer.ts
 import { createWriteStream } from "node:fs";
 import { mkdir as mkdir4 } from "node:fs/promises";
-import path7 from "node:path";
+import path8 from "node:path";
 import { finished } from "node:stream/promises";
 var JsonlWriter = class _JsonlWriter {
   stream;
@@ -1500,7 +1618,7 @@ var JsonlWriter = class _JsonlWriter {
     this.stream = stream;
   }
   static async open(filePath) {
-    await mkdir4(path7.dirname(filePath), { recursive: true });
+    await mkdir4(path8.dirname(filePath), { recursive: true });
     const stream = createWriteStream(filePath, { flags: "w", encoding: "utf8" });
     return new _JsonlWriter(stream);
   }
@@ -1532,7 +1650,7 @@ var JsonlWriter = class _JsonlWriter {
 
 // src/commands/eval/junit-writer.ts
 import { mkdir as mkdir5, writeFile as writeFile5 } from "node:fs/promises";
-import path8 from "node:path";
+import path9 from "node:path";
 function escapeXml(str) {
   return str.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
 }
@@ -1544,7 +1662,7 @@ var JunitWriter = class _JunitWriter {
     this.filePath = filePath;
   }
   static async open(filePath) {
-    await mkdir5(path8.dirname(filePath), { recursive: true });
+    await mkdir5(path9.dirname(filePath), { recursive: true });
     return new _JunitWriter(filePath);
   }
   async append(result) {
@@ -1613,7 +1731,7 @@ ${suiteXmls.join("\n")}
 // src/commands/eval/yaml-writer.ts
 import { createWriteStream as createWriteStream2 } from "node:fs";
 import { mkdir as mkdir6 } from "node:fs/promises";
-import path9 from "node:path";
+import path10 from "node:path";
 import { finished as finished2 } from "node:stream/promises";
 import { stringify as stringifyYaml } from "yaml";
 var YamlWriter = class _YamlWriter {
@@ -1625,7 +1743,7 @@ var YamlWriter = class _YamlWriter {
     this.stream = stream;
   }
   static async open(filePath) {
-    await mkdir6(path9.dirname(filePath), { recursive: true });
+    await mkdir6(path10.dirname(filePath), { recursive: true });
     const stream = createWriteStream2(filePath, { flags: "w", encoding: "utf8" });
     return new _YamlWriter(stream);
   }
@@ -1681,7 +1799,7 @@ async function createOutputWriter(filePath, format) {
 }
 var SUPPORTED_EXTENSIONS = /* @__PURE__ */ new Set([".jsonl", ".json", ".xml", ".yaml", ".yml", ".html", ".htm"]);
 function createWriterFromPath(filePath) {
-  const ext = path10.extname(filePath).toLowerCase();
+  const ext = path11.extname(filePath).toLowerCase();
   switch (ext) {
     case ".jsonl":
       return JsonlWriter.open(filePath);
@@ -1788,12 +1906,12 @@ var ProgressDisplay = class {
   }
   addLogPaths(paths, provider) {
     const newPaths = [];
-    for (const path14 of paths) {
-      if (this.logPathSet.has(path14)) {
+    for (const path16 of paths) {
+      if (this.logPathSet.has(path16)) {
         continue;
       }
-      this.logPathSet.add(path14);
-      newPaths.push(path14);
+      this.logPathSet.add(path16);
+      newPaths.push(path16);
     }
     if (newPaths.length === 0) {
       return;
@@ -1806,8 +1924,8 @@ var ProgressDisplay = class {
       this.hasPrintedLogHeader = true;
     }
     const startIndex = this.logPaths.length - newPaths.length;
-    newPaths.forEach((path14, offset) => {
-      console.log(`${startIndex + offset + 1}. ${path14}`);
+    newPaths.forEach((path16, offset) => {
+      console.log(`${startIndex + offset + 1}. ${path16}`);
     });
   }
   finish() {
@@ -1818,81 +1936,198 @@ var ProgressDisplay = class {
   }
 };
 
-// src/commands/eval/retry-errors.ts
-import { createReadStream } from "node:fs";
-import { createInterface } from "node:readline";
-function getTestId(result) {
-  return result.testId ?? result.test_id;
+// src/commands/results/manifest.ts
+import { existsSync as existsSync2, readFileSync } from "node:fs";
+import path12 from "node:path";
+function parseJsonlLines(content) {
+  return content.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0).map((line) => JSON.parse(line));
 }
-function getExecutionStatus(result) {
-  return result.executionStatus ?? result.execution_status;
+function isIndexManifestPath(sourceFile) {
+  return path12.basename(sourceFile) === RESULT_INDEX_FILENAME;
 }
-function toEvaluationResult(result) {
-  if (result.testId !== void 0 && result.executionStatus !== void 0) {
-    return result;
+function parseMarkdownMessages(content) {
+  const trimmed = content.trim();
+  if (!trimmed.startsWith("@[")) {
+    return [];
+  }
+  const matches = [...trimmed.matchAll(/^@\[(.+?)\]:\n([\s\S]*?)(?=^@\[(.+?)\]:\n|\s*$)/gm)];
+  return matches.map((match) => ({
+    role: match[1],
+    content: match[2].trimEnd()
+  }));
+}
+function readOptionalText(baseDir, relativePath) {
+  if (!relativePath) {
+    return void 0;
+  }
+  const absolutePath = path12.join(baseDir, relativePath);
+  if (!existsSync2(absolutePath)) {
+    return void 0;
+  }
+  return readFileSync(absolutePath, "utf8");
+}
+function readOptionalJson(baseDir, relativePath) {
+  const text = readOptionalText(baseDir, relativePath);
+  if (!text) {
+    return void 0;
   }
+  try {
+    return JSON.parse(text);
+  } catch {
+    return void 0;
+  }
+}
+function hydrateInput(baseDir, record) {
+  const inputText = readOptionalText(baseDir, record.input_path);
+  if (!inputText) {
+    return void 0;
+  }
+  const messages = parseMarkdownMessages(inputText);
+  return messages.length > 0 ? messages : [{ role: "user", content: inputText.trimEnd() }];
+}
+function hydrateOutput(baseDir, record) {
+  const responseText = readOptionalText(baseDir, record.output_path ?? record.response_path);
+  if (!responseText) {
+    return void 0;
+  }
+  const messages = parseMarkdownMessages(responseText);
+  if (messages.length > 0) {
+    return messages.map((message) => ({
+      role: message.role,
+      content: message.content
+    }));
+  }
+  return [{ role: "assistant", content: responseText.trimEnd() }];
+}
+function hydrateManifestRecord(baseDir, record) {
+  const grading = readOptionalJson(baseDir, record.grading_path);
+  const timing = readOptionalJson(baseDir, record.timing_path);
+  const testId = record.test_id ?? record.eval_id ?? "unknown";
   return {
-    ...result,
-    testId: getTestId(result) ?? "",
-    executionStatus: getExecutionStatus(result)
+    timestamp: record.timestamp,
+    testId,
+    eval_set: record.eval_set,
+    target: record.target,
+    score: record.score,
+    executionStatus: record.execution_status,
+    error: record.error,
+    assertions: grading?.assertions.map((assertion) => ({
+      text: assertion.text,
+      passed: assertion.passed,
+      evidence: assertion.evidence
+    })),
+    scores: grading?.evaluators?.map((evaluator) => ({
+      name: evaluator.name,
+      type: evaluator.type,
+      score: evaluator.score,
+      assertions: Array.isArray(evaluator.assertions) ? evaluator.assertions.map((assertion) => ({
+        text: String(assertion.text ?? ""),
+        passed: Boolean(assertion.passed),
+        evidence: typeof assertion.evidence === "string" ? String(assertion.evidence) : void 0
+      })) : void 0,
+      weight: typeof evaluator.weight === "number" ? evaluator.weight : void 0,
+      verdict: typeof evaluator.verdict === "string" ? evaluator.verdict : void 0,
+      details: evaluator.details
+    })) ?? record.scores,
+    tokenUsage: timing?.token_usage ? {
+      input: timing.token_usage.input,
+      output: timing.token_usage.output,
+      reasoning: timing.token_usage.reasoning
+    } : record.token_usage,
+    durationMs: timing?.duration_ms ?? record.duration_ms,
+    costUsd: record.cost_usd,
+    input: hydrateInput(baseDir, record),
+    output: hydrateOutput(baseDir, record)
   };
 }
-async function loadErrorTestIds(jsonlPath) {
-  const ids = [];
-  const rl = createInterface({
-    input: createReadStream(jsonlPath),
-    crlfDelay: Number.POSITIVE_INFINITY
-  });
-  for await (const line of rl) {
+function parseResultManifest(content) {
+  return parseJsonlLines(content);
+}
+function resolveResultSourcePath(source, cwd) {
+  const resolved = path12.isAbsolute(source) ? source : path12.resolve(cwd ?? process.cwd(), source);
+  return resolveWorkspaceOrFilePath(resolved);
+}
+function loadManifestResults(sourceFile) {
+  const resolvedSourceFile = resolveWorkspaceOrFilePath(sourceFile);
+  if (!isIndexManifestPath(resolvedSourceFile)) {
+    return parseJsonlResults(readFileSync(resolvedSourceFile, "utf8"));
+  }
+  const content = readFileSync(resolvedSourceFile, "utf8");
+  const records = parseResultManifest(content);
+  const baseDir = path12.dirname(resolvedSourceFile);
+  return records.map((record) => hydrateManifestRecord(baseDir, record));
+}
+function loadLightweightResults(sourceFile) {
+  const resolvedSourceFile = resolveWorkspaceOrFilePath(sourceFile);
+  const content = readFileSync(resolvedSourceFile, "utf8");
+  if (isIndexManifestPath(resolvedSourceFile)) {
+    return parseResultManifest(content).map((record) => ({
+      testId: record.test_id ?? record.eval_id ?? "unknown",
+      target: record.target,
+      score: record.score,
+      scores: record.scores,
+      executionStatus: record.execution_status,
+      error: record.error,
+      timestamp: record.timestamp
+    }));
+  }
+  const records = [];
+  for (const line of content.split(/\r?\n/)) {
     const trimmed = line.trim();
-    if (!trimmed) continue;
+    if (!trimmed) {
+      continue;
+    }
+    let record;
     try {
-      const parsed = JSON.parse(trimmed);
-      const executionStatus = getExecutionStatus(parsed);
-      const testId = getTestId(parsed);
-      if (executionStatus === "execution_error" && testId) {
-        ids.push(testId);
-      }
+      record = JSON.parse(trimmed);
     } catch {
+      continue;
     }
+    const rawTestId = record.test_id ?? record.eval_id ?? record.testId ?? record.evalId;
+    if (typeof rawTestId !== "string") {
+      throw new Error(`Missing test_id in result: ${trimmed}`);
+    }
+    if (typeof record.score !== "number") {
+      throw new Error(`Missing or invalid score in result: ${trimmed}`);
+    }
+    records.push({
+      testId: rawTestId,
+      target: typeof record.target === "string" ? record.target : void 0,
+      score: record.score,
+      scores: Array.isArray(record.scores) ? record.scores : void 0,
+      executionStatus: typeof record.execution_status === "string" ? record.execution_status : typeof record.executionStatus === "string" ? record.executionStatus : void 0,
+      error: typeof record.error === "string" ? record.error : void 0,
+      timestamp: typeof record.timestamp === "string" ? record.timestamp : void 0
+    });
   }
+  return records;
+}
+
+// src/commands/eval/retry-errors.ts
+async function loadErrorTestIds(jsonlPath) {
+  const resolvedPath = resolveResultSourcePath(jsonlPath);
+  const ids = loadLightweightResults(resolvedPath).filter((result) => result.executionStatus === "execution_error").map((result) => result.testId);
   return [...new Set(ids)];
 }
 async function loadNonErrorResults(jsonlPath) {
-  const results = [];
-  const rl = createInterface({
-    input: createReadStream(jsonlPath),
-    crlfDelay: Number.POSITIVE_INFINITY
-  });
-  for await (const line of rl) {
-    const trimmed = line.trim();
-    if (!trimmed) continue;
-    try {
-      const parsed = JSON.parse(trimmed);
-      const testId = getTestId(parsed);
-      const executionStatus = getExecutionStatus(parsed);
-      if (!testId || parsed.score === void 0) continue;
-      if (executionStatus !== "execution_error") {
-        results.push(toEvaluationResult(parsed));
-      }
-    } catch {
-    }
-  }
-  return results;
+  const resolvedPath = resolveResultSourcePath(jsonlPath);
+  return loadManifestResults(resolvedPath).filter(
+    (result) => result.testId && result.executionStatus !== "execution_error"
+  );
 }
 
 // src/commands/eval/run-cache.ts
 import { mkdir as mkdir7, readFile as readFile2, writeFile as writeFile6 } from "node:fs/promises";
-import path11 from "node:path";
+import path13 from "node:path";
 var CACHE_FILENAME = "cache.json";
 function resolveRunCacheFile(cache) {
   if (cache.lastRunDir) {
-    return path11.join(cache.lastRunDir, "results.jsonl");
+    return resolveExistingRunPrimaryPath(cache.lastRunDir) ?? resolveRunIndexPath(cache.lastRunDir);
   }
   return cache.lastResultFile ?? "";
 }
 function cachePath(cwd) {
-  return path11.join(cwd, ".agentv", CACHE_FILENAME);
+  return path13.join(cwd, ".agentv", CACHE_FILENAME);
 }
 async function loadRunCache(cwd) {
   try {
@@ -1902,11 +2137,15 @@ async function loadRunCache(cwd) {
     return void 0;
   }
 }
-async function saveRunCache(cwd, runDir) {
-  const dir = path11.join(cwd, ".agentv");
+async function saveRunCache(cwd, resultPath) {
+  const dir = path13.join(cwd, ".agentv");
   await mkdir7(dir, { recursive: true });
-  const cache = {
-    lastRunDir: runDir,
+  const basename = path13.basename(resultPath);
+  const cache = basename === RESULT_INDEX_FILENAME || basename === LEGACY_RESULTS_FILENAME ? {
+    lastRunDir: path13.dirname(resultPath),
+    timestamp: (/* @__PURE__ */ new Date()).toISOString()
+  } : {
+    lastResultFile: resultPath,
     timestamp: (/* @__PURE__ */ new Date()).toISOString()
   };
   await writeFile6(cachePath(cwd), `${JSON.stringify(cache, null, 2)}
@@ -2162,7 +2401,7 @@ function formatMatrixSummary(results) {
 
 // ../../packages/core/dist/evaluation/validation/index.js
 import { readFile as readFile3 } from "node:fs/promises";
-import path12 from "node:path";
+import path14 from "node:path";
 import { parse } from "yaml";
 import { readFile as readFile22 } from "node:fs/promises";
 import path22 from "node:path";
@@ -2205,8 +2444,8 @@ async function detectFileType(filePath) {
   }
 }
 function inferFileTypeFromPath(filePath) {
-  const normalized = path12.normalize(filePath).replace(/\\/g, "/");
-  const basename = path12.basename(filePath);
+  const normalized = path14.normalize(filePath).replace(/\\/g, "/");
+  const basename = path14.basename(filePath);
   if (normalized.includes("/.agentv/")) {
     if (basename === "config.yaml" || basename === "config.yml") {
       return "config";
@@ -3834,11 +4073,9 @@ async function ensureFileExists(filePath, description) {
   }
 }
 function buildDefaultOutputPath(cwd) {
-  const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
-  const dirName = `eval_${timestamp}`;
-  const runDir = path13.join(cwd, ".agentv", "results", "raw", dirName);
+  const runDir = buildDefaultRunDir(cwd);
   mkdirSync(runDir, { recursive: true });
-  return path13.join(runDir, "results.jsonl");
+  return path15.join(runDir, "index.jsonl");
 }
 function createProgressReporter(maxWorkers, options) {
   const display = new ProgressDisplay(maxWorkers, options);
@@ -3852,7 +4089,7 @@ function createProgressReporter(maxWorkers, options) {
   };
 }
 function makeEvalKey(testFilePath, evalId) {
-  return `${path13.resolve(testFilePath)}::${evalId}`;
+  return `${path15.resolve(testFilePath)}::${evalId}`;
 }
 function createDisplayIdTracker() {
   const map = /* @__PURE__ */ new Map();
@@ -4041,7 +4278,8 @@ async function runSingleEvalFile(params) {
       vscodeCmd: vsConfig.executable
     });
   }
-  const streamingObserver = otelExporter?.createStreamingObserver() ?? null;
+  const useStreamingObserver = !!(otelExporter && options.exportOtel);
+  const streamingObserver = useStreamingObserver ? otelExporter?.createStreamingObserver() ?? null : null;
   const results = await evaluationRunner({
     testFilePath,
     repoRoot,
@@ -4074,6 +4312,7 @@ async function runSingleEvalFile(params) {
     model: options.model,
     streamCallbacks: streamingObserver?.getStreamCallbacks(),
     onResult: async (result) => {
+      streamingObserver?.completeFromResult?.(result);
       streamingObserver?.finalizeEvalCase(result.score, result.error);
       const trimmedOutput = trimOutputMessages(result.output, options.outputMessages);
       const trimmedResult = {
@@ -4134,7 +4373,7 @@ async function runEvalCommand(input) {
     );
   }
   const repoRoot = await findRepoRoot(cwd);
-  const yamlConfig = await loadConfig(path13.join(cwd, "_"), repoRoot);
+  const yamlConfig = await loadConfig(path15.join(cwd, "_"), repoRoot);
   if (yamlConfig?.required_version) {
     await enforceRequiredVersion(yamlConfig.required_version, {
       strict: normalizeBoolean(input.rawOptions.strict)
@@ -4146,7 +4385,7 @@ async function runEvalCommand(input) {
   }
   let retryNonErrorResults;
   if (options.retryErrors) {
-    const retryPath = path13.resolve(options.retryErrors);
+    const retryPath = path15.resolve(options.retryErrors);
     await ensureFileExists(retryPath, "Retry-errors JSONL file");
     const errorIds = await loadErrorTestIds(retryPath);
     if (errorIds.length === 0) {
@@ -4159,7 +4398,7 @@ async function runEvalCommand(input) {
     retryNonErrorResults = await loadNonErrorResults(retryPath);
   }
   if (options.workspacePath) {
-    const resolvedWorkspace = path13.resolve(options.workspacePath);
+    const resolvedWorkspace = path15.resolve(options.workspacePath);
     try {
       const { stat: stat2 } = await import("node:fs/promises");
       const stats = await stat2(resolvedWorkspace);
@@ -4177,11 +4416,15 @@ async function runEvalCommand(input) {
   if (options.verbose) {
     console.log(`Repository root: ${repoRoot}`);
   }
+  const usesDefaultArtifactWorkspace = !options.outPath;
+  const outputPath = options.outPath ? path15.resolve(options.outPath) : buildDefaultOutputPath(cwd);
+  const defaultTraceFile = usesDefaultArtifactWorkspace && !options.traceFile ? path15.join(path15.dirname(outputPath), "trace.jsonl") : void 0;
+  const traceFilePath = options.traceFile ? path15.resolve(options.traceFile) : defaultTraceFile;
   let otelExporter = null;
-  const useFileExport = !!(options.otelFile || options.traceFile);
+  const useFileExport = !!(options.otelFile || traceFilePath);
   if (options.exportOtel || useFileExport) {
     try {
-      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-QERRYDSC.js");
+      const { OtelTraceExporter, OTEL_BACKEND_PRESETS } = await import("./dist-L6R5HJ72.js");
       let endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
       let headers = {};
       if (options.otelBackend) {
@@ -4205,8 +4448,8 @@ async function runEvalCommand(input) {
         headers,
         captureContent,
         groupTurns: options.otelGroupTurns,
-        otlpFilePath: options.otelFile ? path13.resolve(options.otelFile) : void 0,
-        traceFilePath: options.traceFile ? path13.resolve(options.traceFile) : void 0
+        otlpFilePath: options.otelFile ? path15.resolve(options.otelFile) : void 0,
+        traceFilePath
       });
       const initialized = await otelExporter.init();
       if (!initialized) {
@@ -4222,27 +4465,29 @@ async function runEvalCommand(input) {
       otelExporter = null;
     }
   }
-  const outputPath = options.outPath ? path13.resolve(options.outPath) : buildDefaultOutputPath(cwd);
-  const extraOutputPaths = options.outputPaths.map((p) => path13.resolve(p));
-  const allOutputPaths = extraOutputPaths.length > 0 ? [outputPath, ...extraOutputPaths] : [outputPath];
+  const primaryWritePath = usesDefaultArtifactWorkspace ? path15.join(path15.dirname(outputPath), LEGACY_RESULTS_FILENAME) : outputPath;
+  const extraOutputPaths = options.outputPaths.map((p) => path15.resolve(p));
+  const allOutputPaths = extraOutputPaths.length > 0 ? [primaryWritePath, ...extraOutputPaths] : [primaryWritePath];
   const uniqueOutputPaths = [...new Set(allOutputPaths)];
+  const reportedOutputPaths = extraOutputPaths.length > 0 ? [outputPath, ...extraOutputPaths] : [outputPath];
+  const uniqueReportedOutputPaths = [...new Set(reportedOutputPaths)];
   let outputWriter;
   if (uniqueOutputPaths.length === 1) {
-    outputWriter = await createOutputWriter(outputPath, options.format);
+    outputWriter = await createOutputWriter(primaryWritePath, options.format);
     console.log(`Output path: ${outputPath}`);
   } else {
     outputWriter = await createMultiWriter(uniqueOutputPaths);
     console.log("Output paths:");
-    for (const p of uniqueOutputPaths) {
+    for (const p of uniqueReportedOutputPaths) {
       console.log(`  ${p}`);
     }
   }
-  const resolvedTestFiles = input.testFiles.map((file) => path13.resolve(file));
+  const resolvedTestFiles = input.testFiles.map((file) => path15.resolve(file));
   if (options.otelFile) {
-    console.log(`OTLP JSON file: ${path13.resolve(options.otelFile)}`);
+    console.log(`OTLP JSON file: ${path15.resolve(options.otelFile)}`);
   }
-  if (options.traceFile) {
-    console.log(`Trace file: ${path13.resolve(options.traceFile)}`);
+  if (traceFilePath) {
+    console.log(`Trace file: ${traceFilePath}`);
   }
   const evaluationRunner = await resolveEvaluationRunner();
   const allResults = [];
@@ -4288,7 +4533,7 @@ async function runEvalCommand(input) {
     cliNoCache: options.noCache,
     yamlCache: yamlCacheEnabled
   });
-  const cache = cacheEnabled ? new ResponseCache(yamlCachePath ? path13.resolve(yamlCachePath) : void 0) : void 0;
+  const cache = cacheEnabled ? new ResponseCache(yamlCachePath ? path15.resolve(yamlCachePath) : void 0) : void 0;
   const useCache = cacheEnabled;
   if (cacheEnabled) {
     console.log(`Response cache: enabled${yamlCachePath ? ` (${yamlCachePath})` : ""}`);
@@ -4420,20 +4665,51 @@ async function runEvalCommand(input) {
       console.log(formatMatrixSummary(allResults));
     }
     if (options.benchmarkJson && allResults.length > 0) {
-      const benchmarkPath = path13.resolve(options.benchmarkJson);
+      const benchmarkPath = path15.resolve(options.benchmarkJson);
       await writeBenchmarkJson(benchmarkPath, allResults);
       console.log(`Benchmark written to: ${benchmarkPath}`);
     }
-    if (options.artifacts && allResults.length > 0) {
-      const artifactsDir = path13.resolve(options.artifacts);
+    if (usesDefaultArtifactWorkspace) {
       const evalFile = resolvedTestFiles.length === 1 ? resolvedTestFiles[0] : "";
+      const workspaceDir = path15.dirname(outputPath);
       const {
-        gradingDir,
+        testArtifactDir,
+        timingPath,
+        benchmarkPath: workspaceBenchmarkPath,
+        indexPath,
+        legacyResultsPath
+      } = await writeArtifactsFromResults(allResults, workspaceDir, {
+        evalFile,
+        writeLegacyResults: true
+      });
+      console.log(`Artifact workspace written to: ${workspaceDir}`);
+      console.log(`  Index: ${indexPath}`);
+      console.log(
+        `  Per-test artifacts: ${testArtifactDir} (${allResults.length} test directories)`
+      );
+      console.log(`  Timing: ${timingPath}`);
+      console.log(`  Benchmark: ${workspaceBenchmarkPath}`);
+      if (legacyResultsPath) {
+        console.log(`  Compatibility output: ${legacyResultsPath} (deprecated)`);
+      }
+    }
+    if (options.artifacts) {
+      const artifactsDir = path15.resolve(options.artifacts);
+      const evalFile = resolvedTestFiles.length === 1 ? resolvedTestFiles[0] : "";
+      const {
+        testArtifactDir,
+        indexPath,
         timingPath,
         benchmarkPath: abp
-      } = await writeArtifactsFromResults(allResults, artifactsDir, { evalFile });
+      } = await writeArtifactsFromResults(allResults, artifactsDir, {
+        evalFile,
+        writeLegacyResults: false
+      });
       console.log(`Artifacts written to: ${artifactsDir}`);
-      console.log(`  Grading: ${gradingDir} (${allResults.length} files)`);
+      console.log(`  Index: ${indexPath}`);
+      console.log(
+        `  Per-test artifacts: ${testArtifactDir} (${allResults.length} test directories)`
+      );
       console.log(`  Timing:  ${timingPath}`);
       console.log(`  Benchmark: ${abp}`);
     }
@@ -4447,33 +4723,25 @@ async function runEvalCommand(input) {
       }
     }
     if (allResults.length > 0) {
-      if (uniqueOutputPaths.length === 1) {
+      if (uniqueReportedOutputPaths.length === 1) {
         console.log(`
 Results written to: ${outputPath}`);
       } else {
         console.log("\nResults written to:");
-        for (const p of uniqueOutputPaths) {
+        for (const p of uniqueReportedOutputPaths) {
           console.log(`  ${p}`);
         }
       }
-      const runDir = path13.dirname(outputPath);
-      await saveRunCache(cwd, runDir).catch(() => void 0);
-      if (outputPath.endsWith(".jsonl")) {
-        const { writeFile: writeFile7 } = await import("node:fs/promises");
-        const gradingPath = path13.join(path13.dirname(outputPath), "grading.json");
-        const aggregateGrading = buildAggregateGradingArtifact(allResults);
-        await writeFile7(gradingPath, `${JSON.stringify(aggregateGrading, null, 2)}
-`, "utf8");
-      }
+      await saveRunCache(cwd, outputPath).catch(() => void 0);
     }
     if (summary.executionErrorCount > 0 && !options.retryErrors) {
-      const evalFileArgs = resolvedTestFiles.map((f) => path13.relative(cwd, f)).join(" ");
+      const evalFileArgs = resolvedTestFiles.map((f) => path15.relative(cwd, f)).join(" ");
       const targetFlag = options.target ? ` --target ${options.target}` : "";
-      const relativeOutputPath = path13.relative(cwd, outputPath);
+      const relativeOutputPath = path15.relative(cwd, outputPath);
       console.log(
         `
 Tip: ${summary.executionErrorCount} execution error(s) detected. Re-run failed tests with:
-  agentv eval run ${evalFileArgs}${targetFlag} --retry-errors ${relativeOutputPath} -o ${relativeOutputPath}`
+  agentv eval run ${evalFileArgs}${targetFlag} --retry-errors ${relativeOutputPath}`
       );
     }
     return {
@@ -4501,7 +4769,7 @@ async function resolveEvaluationRunner() {
   if (!overridePath) {
     return runEvaluation;
   }
-  const resolved = path13.isAbsolute(overridePath) ? overridePath : path13.resolve(process.cwd(), overridePath);
+  const resolved = path15.isAbsolute(overridePath) ? overridePath : path15.resolve(process.cwd(), overridePath);
   const moduleUrl = pathToFileURL(resolved).href;
   const mod = await import(moduleUrl);
   const candidate = mod.runEvaluation;
@@ -4516,14 +4784,17 @@ async function resolveEvaluationRunner() {
 export {
   package_default,
   toSnakeCaseDeep,
+  RESULT_INDEX_FILENAME,
+  LEGACY_RESULTS_FILENAME,
+  resolveExistingRunPrimaryPath,
+  resolveWorkspaceOrFilePath,
+  writeArtifactsFromResults,
+  resolveResultSourcePath,
+  loadManifestResults,
+  loadLightweightResults,
   HtmlWriter,
   resolveEvalPaths,
   findRepoRoot,
-  buildGradingArtifact,
-  buildTimingArtifact,
-  buildBenchmarkArtifact,
-  buildAggregateGradingArtifact,
-  parseJsonlResults,
   resolveRunCacheFile,
   loadRunCache,
   detectFileType,
@@ -4536,4 +4807,4 @@ export {
   selectTarget,
   runEvalCommand
 };
-//# sourceMappingURL=chunk-JEW3FEO7.js.map
+//# sourceMappingURL=chunk-VLOFRXH4.js.map