agentv 4.2.0 → 4.3.0

package/dist/{chunk-UXSQQHCI.js → chunk-ASU5L5ZW.js}

@@ -24,7 +24,7 @@ import {
   validateFileReferences,
   validateTargetsFile,
   writeArtifactsFromResults
-} from "./chunk-HAZJO7OY.js";
+} from "./chunk-ZDJN5FSI.js";
 import {
   DEFAULT_CATEGORY,
   createBuiltinRegistry,
@@ -4217,7 +4217,7 @@ var evalRunCommand = command({
   },
   handler: async (args) => {
     if (args.evalPaths.length === 0 && process.stdin.isTTY) {
-      const { launchInteractiveWizard } = await import("./interactive-NVNOLL2H.js");
+      const { launchInteractiveWizard } = await import("./interactive-BKK53ETJ.js");
       await launchInteractiveWizard();
       return;
     }
@@ -4441,27 +4441,15 @@ var evalBenchCommand = command({
       type: string,
       displayName: "export-dir",
       description: "Export directory from pipeline input/grade"
-    }),
-    llmScores: option({
-      type: optional(string),
-      long: "llm-scores",
-      description: "Path to LLM scores JSON file (reads from stdin if omitted)"
     })
   },
-  handler: async ({ exportDir, llmScores: llmScoresPath }) => {
+  handler: async ({ exportDir }) => {
     const manifest = JSON.parse(await readFile(join(exportDir, "manifest.json"), "utf8"));
     const testIds = manifest.test_ids;
     const targetName = manifest.target?.name ?? "unknown";
     const evalSet = manifest.dataset ?? "";
     const experiment = manifest.experiment;
     const safeEvalSet = evalSet ? evalSet.replace(/[\/\\:*?"<>|]/g, "_") : "";
-    let stdinData;
-    if (llmScoresPath) {
-      stdinData = await readFile(llmScoresPath, "utf8");
-    } else {
-      stdinData = await readStdin();
-    }
-    const llmScores = stdinData ? JSON.parse(stdinData) : {};
     const indexLines = [];
     const allPassRates = [];
     for (const testId of testIds) {
@@ -4488,14 +4476,18 @@ var evalBenchCommand = command({
         }
       } catch {
       }
-      const testLlmScores = llmScores[testId] ?? {};
       const llmGradersDir = join(testDir, "llm_graders");
       try {
         const graderFiles = (await readdir(llmGradersDir)).filter((f) => f.endsWith(".json"));
         for (const file of graderFiles) {
           const graderMeta = JSON.parse(await readFile(join(llmGradersDir, file), "utf8"));
           const graderName = graderMeta.name;
-          const llmResult = testLlmScores[graderName];
+          const diskResultPath = join(testDir, "llm_grader_results", `${graderName}.json`);
+          let llmResult;
+          try {
+            llmResult = JSON.parse(await readFile(diskResultPath, "utf8"));
+          } catch {
+          }
           if (llmResult) {
             evaluators.push({
               name: graderName,
@@ -4515,7 +4507,7 @@ var evalBenchCommand = command({
       const weightedScore = totalWeight > 0 ? evaluators.reduce((sum, e) => sum + e.score * e.weight, 0) / totalWeight : 0;
       const passed = allAssertions.filter((a) => a.passed).length;
       const failed = allAssertions.filter((a) => !a.passed).length;
-      const passRate = allAssertions.length > 0 ? Math.round(passed / allAssertions.length * 1e3) / 1e3 : 0;
+      const passRate = allAssertions.length > 0 ? Math.round(passed / allAssertions.length * 1e3) / 1e3 : weightedScore >= 0.5 ? 1 : 0;
       allPassRates.push(passRate);
       const grading = {
         assertions: allAssertions,
@@ -4608,13 +4600,6 @@ var evalBenchCommand = command({
     console.log(`Benchmark: ${testIds.length} test(s), pass_rate=${passRateStats.mean}`);
   }
 });
-async function readStdin() {
-  const chunks = [];
-  for await (const chunk of process.stdin) {
-    chunks.push(chunk);
-  }
-  return Buffer.concat(chunks).toString("utf8").trim();
-}
 function computeStats(values) {
   if (values.length === 0) return { mean: 0, stddev: 0 };
   const mean2 = values.reduce((sum, v) => sum + v, 0) / values.length;
@@ -4681,7 +4666,10 @@ async function runCodeGraders(tasks, concurrency) {
       );
       const parsed = JSON.parse(stdout);
       const score = typeof parsed.score === "number" ? parsed.score : 0;
-      const assertions = Array.isArray(parsed.assertions) ? parsed.assertions : [];
+      const assertions = Array.isArray(parsed.assertions) && parsed.assertions.length > 0 ? parsed.assertions : [
+        ...(parsed.hits ?? []).map((h) => ({ text: h, passed: true })),
+        ...(parsed.misses ?? []).map((m) => ({ text: m, passed: false }))
+      ];
       const result = {
         name: graderName,
         type: "code-grader",
@@ -8330,4 +8318,4 @@ export {
   preprocessArgv,
   runCli
 };
-//# sourceMappingURL=chunk-UXSQQHCI.js.map
+//# sourceMappingURL=chunk-ASU5L5ZW.js.map