agentv 3.3.0 → 3.4.0

package/README.md

@@ -309,7 +309,7 @@ const { results, summary } = await evaluate({
     {
       id: 'greeting',
       input: 'Say hello',
-      assert: [{ type: 'contains', value: 'Hello' }],
+      assertions: [{ type: 'contains', value: 'Hello' }],
     },
   ],
 });

package/dist/{chunk-6LP5Z5Y4.js → chunk-A7ZDUB46.js}

@@ -1,9 +1,13 @@
 import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
 import {
   HtmlWriter,
+  buildBenchmarkArtifact,
+  buildGradingArtifact,
+  buildTimingArtifact,
   detectFileType,
   findRepoRoot,
   package_default,
+  parseJsonlResults,
   resolveEvalPaths,
   runEvalCommand,
   selectTarget,
@@ -12,7 +16,7 @@ import {
   validateEvalFile,
   validateFileReferences,
   validateTargetsFile
-} from "./chunk-4ZMSAQWS.js";
+} from "./chunk-RE5I3U2S.js";
 import {
   createBuiltinRegistry,
   createProvider,
@@ -30,7 +34,7 @@ import {
   toSnakeCaseDeep as toSnakeCaseDeep2,
   transpileEvalYamlFile,
   trimBaselineResult
-} from "./chunk-5M3K2DMV.js";
+} from "./chunk-GOZV2HN2.js";
 import {
   __commonJS,
   __esm,
@@ -4181,7 +4185,7 @@ var evalRunCommand = command({
   },
   handler: async (args) => {
     if (args.evalPaths.length === 0 && process.stdin.isTTY) {
-      const { launchInteractiveWizard } = await import("./interactive-NA6SAIAG.js");
+      const { launchInteractiveWizard } = await import("./interactive-WXXTZ7PD.js");
       await launchInteractiveWizard();
       return;
     }
@@ -4565,93 +4569,9 @@ var initCmdTsCommand = command({
   }
 });
 
-// src/commands/self/index.ts
-import { spawn } from "node:child_process";
-function detectPackageManagerFromPath(scriptPath) {
-  if (scriptPath.includes(".bun")) {
-    return "bun";
-  }
-  return "npm";
-}
-function detectPackageManager() {
-  return detectPackageManagerFromPath(process.argv[1] ?? "");
-}
-function runCommand(cmd, args) {
-  return new Promise((resolve, reject) => {
-    const child = spawn(cmd, args, { stdio: ["inherit", "pipe", "inherit"], shell: true });
-    let stdout = "";
-    child.stdout?.on("data", (data) => {
-      process.stdout.write(data);
-      stdout += data.toString();
-    });
-    child.on("error", reject);
-    child.on("close", (code) => resolve({ exitCode: code ?? 1, stdout }));
-  });
-}
-var updateCommand = command({
-  name: "update",
-  description: "Update agentv to the latest version",
-  args: {
-    npm: flag({ long: "npm", description: "Force update using npm" }),
-    bun: flag({ long: "bun", description: "Force update using bun" })
-  },
-  handler: async ({ npm, bun }) => {
-    if (npm && bun) {
-      console.error("Error: Cannot specify both --npm and --bun");
-      process.exit(1);
-    }
-    let pm;
-    if (npm) {
-      pm = "npm";
-    } else if (bun) {
-      pm = "bun";
-    } else {
-      pm = detectPackageManager();
-    }
-    const currentVersion = package_default.version;
-    console.log(`Current version: ${currentVersion}`);
-    console.log(`Updating agentv using ${pm}...
-`);
-    const args = pm === "npm" ? ["install", "-g", "agentv@latest"] : ["add", "-g", "agentv@latest"];
-    try {
-      const result = await runCommand(pm, args);
-      if (result.exitCode !== 0) {
-        console.error("\nUpdate failed.");
-        process.exit(1);
-      }
-      let newVersion;
-      try {
-        const versionResult = await runCommand("agentv", ["--version"]);
-        newVersion = versionResult.stdout.trim();
-      } catch {
-      }
-      if (newVersion) {
-        console.log(`
-Update complete: ${currentVersion} \u2192 ${newVersion}`);
-      } else {
-        console.log("\nUpdate complete.");
-      }
-    } catch (error) {
-      if (error instanceof Error) {
-        if (error.message.includes("ENOENT") || error.message.includes("not found")) {
-          const alternative = pm === "npm" ? "bun" : "npm";
-          console.error(`Error: ${pm} not found. Try using --${alternative} flag.`);
-        } else {
-          console.error(`Error: ${error.message}`);
-        }
-        process.exit(1);
-      }
-      throw error;
-    }
-  }
-});
-var selfCommand = subcommands({
-  name: "self",
-  description: "Manage the agentv installation",
-  cmds: {
-    update: updateCommand
-  }
-});
+// src/commands/results/export.ts
+import { mkdirSync as mkdirSync2, readFileSync as readFileSync6, writeFileSync as writeFileSync3 } from "node:fs";
+import path8 from "node:path";
 
 // src/commands/trace/utils.ts
 import { readFileSync as readFileSync5, readdirSync as readdirSync2, statSync as statSync2 } from "node:fs";
@@ -4757,6 +4677,202 @@ function formatScore(score) {
   return `${(score * 100).toFixed(0)}%`;
 }
 
+// src/commands/results/export.ts
+function exportResults(sourceFile, content, outputDir) {
+  const results = parseJsonlResults(content);
+  if (results.length === 0) {
+    throw new Error(`No results found in ${sourceFile}`);
+  }
+  const patched = results.map((r) => {
+    if (!r.testId && r.evalId) {
+      return { ...r, testId: String(r.evalId) };
+    }
+    return r;
+  });
+  mkdirSync2(outputDir, { recursive: true });
+  const benchmark = buildBenchmarkArtifact(patched, sourceFile);
+  writeFileSync3(path8.join(outputDir, "benchmark.json"), `${JSON.stringify(benchmark, null, 2)}
+`);
+  const timing = buildTimingArtifact(patched);
+  writeFileSync3(path8.join(outputDir, "timing.json"), `${JSON.stringify(timing, null, 2)}
+`);
+  const gradingDir = path8.join(outputDir, "grading");
+  mkdirSync2(gradingDir, { recursive: true });
+  for (const result of patched) {
+    const id = safeTestId(result);
+    const grading = buildGradingArtifact(result);
+    writeFileSync3(path8.join(gradingDir, `${id}.json`), `${JSON.stringify(grading, null, 2)}
+`);
+  }
+  const outputsDir = path8.join(outputDir, "outputs");
+  mkdirSync2(outputsDir, { recursive: true });
+  for (const result of patched) {
+    const answer = result.answer;
+    if (answer) {
+      const id = safeTestId(result);
+      writeFileSync3(path8.join(outputsDir, `${id}.txt`), answer);
+    }
+  }
+}
+function safeTestId(result) {
+  const raw = result.testId ?? result.evalId ?? "unknown";
+  return String(raw).replace(/[/\\:*?"<>|]/g, "_");
+}
+function deriveOutputDir(cwd, sourceFile) {
+  const basename = path8.basename(sourceFile, ".jsonl");
+  const dirName = basename.startsWith("eval_") ? basename.slice(5) : basename;
+  return path8.join(cwd, ".agentv", "results", dirName);
+}
+var resultsExportCommand = command({
+  name: "export",
+  description: "Export JSONL eval results into a per-test directory structure",
+  args: {
+    source: positional({
+      type: optional(string),
+      displayName: "source",
+      description: "JSONL result file to export (defaults to most recent in .agentv/results/)"
+    }),
+    out: option({
+      type: optional(string),
+      long: "out",
+      short: "o",
+      description: "Output directory (defaults to .agentv/results/<run-timestamp>/)"
+    }),
+    dir: option({
+      type: optional(string),
+      long: "dir",
+      short: "d",
+      description: "Working directory (default: current directory)"
+    })
+  },
+  handler: async ({ source, out, dir }) => {
+    const cwd = dir ?? process.cwd();
+    try {
+      let sourceFile;
+      if (source) {
+        sourceFile = path8.isAbsolute(source) ? source : path8.resolve(cwd, source);
+      } else {
+        const metas = listResultFiles(cwd, 1);
+        if (metas.length === 0) {
+          console.error("Error: No result files found in .agentv/results/");
+          console.error("Run an evaluation first: agentv eval <eval-file>");
+          process.exit(1);
+        }
+        sourceFile = metas[0].path;
+      }
+      const content = readFileSync6(sourceFile, "utf8");
+      const outputDir = out ? path8.isAbsolute(out) ? out : path8.resolve(cwd, out) : deriveOutputDir(cwd, sourceFile);
+      exportResults(sourceFile, content, outputDir);
+      const results = parseJsonlResults(content);
+      console.log(`Exported ${results.length} test(s) to ${outputDir}`);
+      for (const result of results) {
+        const id = result.testId ?? result.evalId ?? "unknown";
+        console.log(`  ${id}`);
+      }
+    } catch (error) {
+      console.error(`Error: ${error.message}`);
+      process.exit(1);
+    }
+  }
+});
+
+// src/commands/results/index.ts
+var resultsCommand = subcommands({
+  name: "results",
+  description: "Inspect, export, and manage evaluation results",
+  cmds: {
+    export: resultsExportCommand
+  }
+});
+
+// src/commands/self/index.ts
+import { spawn } from "node:child_process";
+function detectPackageManagerFromPath(scriptPath) {
+  if (scriptPath.includes(".bun")) {
+    return "bun";
+  }
+  return "npm";
+}
+function detectPackageManager() {
+  return detectPackageManagerFromPath(process.argv[1] ?? "");
+}
+function runCommand(cmd, args) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(cmd, args, { stdio: ["inherit", "pipe", "inherit"], shell: true });
+    let stdout = "";
+    child.stdout?.on("data", (data) => {
+      process.stdout.write(data);
+      stdout += data.toString();
+    });
+    child.on("error", reject);
+    child.on("close", (code) => resolve({ exitCode: code ?? 1, stdout }));
+  });
+}
+var updateCommand = command({
+  name: "update",
+  description: "Update agentv to the latest version",
+  args: {
+    npm: flag({ long: "npm", description: "Force update using npm" }),
+    bun: flag({ long: "bun", description: "Force update using bun" })
+  },
+  handler: async ({ npm, bun }) => {
+    if (npm && bun) {
+      console.error("Error: Cannot specify both --npm and --bun");
+      process.exit(1);
+    }
+    let pm;
+    if (npm) {
+      pm = "npm";
+    } else if (bun) {
+      pm = "bun";
+    } else {
+      pm = detectPackageManager();
+    }
+    const currentVersion = package_default.version;
+    console.log(`Current version: ${currentVersion}`);
+    console.log(`Updating agentv using ${pm}...
+`);
+    const args = pm === "npm" ? ["install", "-g", "agentv@latest"] : ["add", "-g", "agentv@latest"];
+    try {
+      const result = await runCommand(pm, args);
+      if (result.exitCode !== 0) {
+        console.error("\nUpdate failed.");
+        process.exit(1);
+      }
+      let newVersion;
+      try {
+        const versionResult = await runCommand("agentv", ["--version"]);
+        newVersion = versionResult.stdout.trim();
+      } catch {
+      }
+      if (newVersion) {
+        console.log(`
+Update complete: ${currentVersion} \u2192 ${newVersion}`);
+      } else {
+        console.log("\nUpdate complete.");
+      }
+    } catch (error) {
+      if (error instanceof Error) {
+        if (error.message.includes("ENOENT") || error.message.includes("not found")) {
+          const alternative = pm === "npm" ? "bun" : "npm";
+          console.error(`Error: ${pm} not found. Try using --${alternative} flag.`);
+        } else {
+          console.error(`Error: ${error.message}`);
+        }
+        process.exit(1);
+      }
+      throw error;
+    }
+  }
+});
+var selfCommand = subcommands({
+  name: "self",
+  description: "Manage the agentv installation",
+  cmds: {
+    update: updateCommand
+  }
+});
+
 // src/commands/trace/list.ts
 function formatListTable(metas) {
   const lines = [];
@@ -5505,8 +5621,8 @@ var traceCommand = subcommands({
 });
 
 // src/commands/transpile/index.ts
-import { writeFileSync as writeFileSync3 } from "node:fs";
-import path8 from "node:path";
+import { writeFileSync as writeFileSync4 } from "node:fs";
+import path9 from "node:path";
 var transpileCommand = command({
   name: "transpile",
   description: "Convert an EVAL.yaml file to Agent Skills evals.json format",
@@ -5530,7 +5646,7 @@ var transpileCommand = command({
   handler: async ({ input, outDir, stdout }) => {
     let result;
     try {
-      result = transpileEvalYamlFile(path8.resolve(input));
+      result = transpileEvalYamlFile(path9.resolve(input));
     } catch (error) {
       console.error(`Error: ${error.message}`);
       process.exit(1);
@@ -5554,12 +5670,12 @@ var transpileCommand = command({
       process.stdout.write("\n");
       return;
     }
-    const outputDir = outDir ? path8.resolve(outDir) : path8.dirname(path8.resolve(input));
+    const outputDir = outDir ? path9.resolve(outDir) : path9.dirname(path9.resolve(input));
     const fileNames = getOutputFilenames(result);
     for (const [skill, evalsJson] of result.files) {
       const fileName = fileNames.get(skill) ?? "evals.json";
-      const outputPath = path8.join(outputDir, fileName);
-      writeFileSync3(outputPath, `${JSON.stringify(evalsJson, null, 2)}
+      const outputPath = path9.join(outputDir, fileName);
+      writeFileSync4(outputPath, `${JSON.stringify(evalsJson, null, 2)}
 `);
       console.log(`Transpiled to ${outputPath}`);
     }
@@ -5567,7 +5683,7 @@ var transpileCommand = command({
 });
 
 // src/commands/trim/index.ts
-import { readFileSync as readFileSync6, writeFileSync as writeFileSync4 } from "node:fs";
+import { readFileSync as readFileSync7, writeFileSync as writeFileSync5 } from "node:fs";
 var trimCommand = command({
   name: "trim",
   description: "Trim evaluation results for baseline storage (strips debug/audit fields)",
@@ -5586,7 +5702,7 @@ var trimCommand = command({
   },
   handler: async ({ input, out }) => {
     try {
-      const content = readFileSync6(input, "utf8");
+      const content = readFileSync7(input, "utf8");
       const lines = content.trim().split("\n").filter((line) => line.trim());
       const trimmedLines = lines.map((line) => {
         const record = JSON.parse(line);
@@ -5598,7 +5714,7 @@ var trimCommand = command({
       const output = `${trimmedLines.join("\n")}
 `;
       if (out) {
-        writeFileSync4(out, output, "utf8");
+        writeFileSync5(out, output, "utf8");
         console.error(`Trimmed ${lines.length} record(s) \u2192 ${out}`);
       } else {
         process.stdout.write(output);
@@ -5693,7 +5809,7 @@ function isTTY() {
 // src/commands/validate/validate-files.ts
 import { constants } from "node:fs";
 import { access, readdir, stat } from "node:fs/promises";
-import path9 from "node:path";
+import path10 from "node:path";
 async function validateFiles(paths) {
   const filePaths = await expandPaths(paths);
   const results = [];
@@ -5711,7 +5827,7 @@ async function validateFiles(paths) {
   };
 }
 async function validateSingleFile(filePath) {
-  const absolutePath = path9.resolve(filePath);
+  const absolutePath = path10.resolve(filePath);
   const fileType = await detectFileType(absolutePath);
   let result;
   if (fileType === "eval") {
@@ -5736,7 +5852,7 @@ async function validateSingleFile(filePath) {
 async function expandPaths(paths) {
   const expanded = [];
   for (const inputPath of paths) {
-    const absolutePath = path9.resolve(inputPath);
+    const absolutePath = path10.resolve(inputPath);
     try {
       await access(absolutePath, constants.F_OK);
     } catch {
@@ -5760,7 +5876,7 @@ async function findYamlFiles(dirPath) {
   try {
     const entries2 = await readdir(dirPath, { withFileTypes: true });
     for (const entry of entries2) {
-      const fullPath = path9.join(dirPath, entry.name);
+      const fullPath = path10.join(dirPath, entry.name);
       if (entry.isDirectory()) {
         if (entry.name === "node_modules" || entry.name.startsWith(".")) {
           continue;
@@ -5777,7 +5893,7 @@ async function findYamlFiles(dirPath) {
   return results;
 }
 function isYamlFile(filePath) {
-  const ext = path9.extname(filePath).toLowerCase();
+  const ext = path10.extname(filePath).toLowerCase();
   return ext === ".yaml" || ext === ".yml";
 }
 
@@ -5817,7 +5933,7 @@ var validateCommand = command({
 // src/commands/workspace/clean.ts
 import { existsSync as existsSync2 } from "node:fs";
 import { readFile as readFile2, readdir as readdir2, rm } from "node:fs/promises";
-import path10 from "node:path";
+import path11 from "node:path";
 async function confirm(message) {
   const readline2 = await import("node:readline");
   const rl = readline2.createInterface({ input: process.stdin, output: process.stdout });
@@ -5853,8 +5969,8 @@ var cleanCommand = command({
       const poolDirs = entries2.filter((e) => e.isDirectory());
       const matchingDirs = [];
       for (const dir of poolDirs) {
-        const poolDir = path10.join(poolRoot, dir.name);
-        const metadataPath = path10.join(poolDir, "metadata.json");
+        const poolDir = path11.join(poolRoot, dir.name);
+        const metadataPath = path11.join(poolDir, "metadata.json");
         try {
           const raw = await readFile2(metadataPath, "utf-8");
           const metadata = JSON.parse(raw);
@@ -5885,7 +6001,7 @@ var cleanCommand = command({
       }
       for (const dir of matchingDirs) {
         await rm(dir, { recursive: true, force: true });
-        console.log(`Removed: ${path10.basename(dir).slice(0, 12)}...`);
+        console.log(`Removed: ${path11.basename(dir).slice(0, 12)}...`);
       }
       console.log("Done.");
     } else {
@@ -5905,13 +6021,13 @@ var cleanCommand = command({
 // src/commands/workspace/list.ts
 import { existsSync as existsSync3 } from "node:fs";
 import { readFile as readFile3, readdir as readdir3, stat as stat2 } from "node:fs/promises";
-import path11 from "node:path";
+import path12 from "node:path";
 async function getDirectorySize(dirPath) {
   let totalSize = 0;
   try {
     const entries2 = await readdir3(dirPath, { withFileTypes: true });
     for (const entry of entries2) {
-      const fullPath = path11.join(dirPath, entry.name);
+      const fullPath = path12.join(dirPath, entry.name);
       if (entry.isDirectory()) {
         totalSize += await getDirectorySize(fullPath);
       } else {
@@ -5946,11 +6062,11 @@ var listCommand = command({
       return;
     }
     for (const dir of poolDirs) {
-      const poolDir = path11.join(poolRoot, dir.name);
+      const poolDir = path12.join(poolRoot, dir.name);
       const fingerprint = dir.name;
       const poolEntries = await readdir3(poolDir, { withFileTypes: true });
       const slots = poolEntries.filter((e) => e.isDirectory() && e.name.startsWith("slot-"));
-      const metadataPath = path11.join(poolDir, "metadata.json");
+      const metadataPath = path12.join(poolDir, "metadata.json");
       let metadata = null;
       try {
         const raw = await readFile3(metadataPath, "utf-8");
@@ -5996,8 +6112,8 @@ var CHECK_INTERVAL_MS = 24 * 60 * 60 * 1e3;
 var AGENTV_DIR = getAgentvHome();
 var CACHE_FILE = "version-check.json";
 var NPM_REGISTRY_URL = "https://registry.npmjs.org/agentv/latest";
-async function getCachedUpdateInfo(path12) {
-  const filePath = path12 ?? join(AGENTV_DIR, CACHE_FILE);
+async function getCachedUpdateInfo(path13) {
+  const filePath = path13 ?? join(AGENTV_DIR, CACHE_FILE);
   try {
     const raw = await readFile4(filePath, "utf-8");
     const data = JSON.parse(raw);
@@ -6087,6 +6203,7 @@ var app = subcommands({
     create: createCommand,
     generate: generateCommand,
     init: initCmdTsCommand,
+    results: resultsCommand,
     self: selfCommand,
     trace: traceCommand,
     transpile: transpileCommand,
@@ -6103,6 +6220,7 @@ var TOP_LEVEL_COMMANDS = /* @__PURE__ */ new Set([
   "create",
   "generate",
   "init",
+  "results",
   "self",
   "trace",
   "transpile",
@@ -6150,4 +6268,4 @@ export {
   preprocessArgv,
   runCli
 };
-//# sourceMappingURL=chunk-6LP5Z5Y4.js.map
+//# sourceMappingURL=chunk-A7ZDUB46.js.map