@agentv/core 4.31.0-next.1 → 4.31.1-next.1

package/dist/index.js

@@ -132,7 +132,7 @@ import {
   tokensPerTool,
   trackChild,
   trackedChildCount
-} from "./chunk-QY6BS2V4.js";
+} from "./chunk-A27NE3R7.js";
 import {
   COMMON_TARGET_SETTINGS,
   TEST_MESSAGE_ROLES,
@@ -720,8 +720,16 @@ function shouldSkipCacheForTemperature(targetConfig) {
 }
 
 // src/evaluation/results-repo.ts
-import { execFile } from "node:child_process";
-import { existsSync, mkdirSync, readFileSync as readFileSync2, rmSync, writeFileSync } from "node:fs";
+import { execFile, spawn } from "node:child_process";
+import {
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync as readFileSync2,
+  renameSync,
+  rmSync,
+  writeFileSync
+} from "node:fs";
 import { cp, mkdtemp, readdir, rm, stat } from "node:fs/promises";
 import os from "node:os";
 import path4 from "node:path";
@@ -738,10 +746,19 @@ function withFriendlyGitHubAuthError(error) {
   }
   return new Error(message);
 }
+function expandHome(p) {
+  if (p === "~" || p.startsWith("~/") || p.startsWith("~\\")) {
+    return path4.join(os.homedir(), p.slice(1));
+  }
+  return p;
+}
 function normalizeResultsConfig(config) {
+  const repo = config.repo.trim();
+  const resolvedPath = config.path ? expandHome(config.path.trim()) : path4.join(getAgentvHome(), "results", sanitizeRepoSlug(repo));
   return {
-    repo: config.repo.trim(),
-    path: config.path.trim().replace(/^\/+|\/+$/g, ""),
+    mode: "github",
+    repo,
+    path: resolvedPath,
     auto_push: config.auto_push === true,
     branch_prefix: config.branch_prefix?.trim() || "eval-results"
   };
@@ -752,7 +769,7 @@ function resolveResultsRepoUrl(repo) {
   }
   return `https://github.com/${repo}.git`;
 }
-function getResultsRepoCachePaths(repo) {
+function getResultsRepoLocalPaths(repo) {
   const rootDir = path4.join(getAgentvHome(), "cache", "results-repo", sanitizeRepoSlug(repo));
   return {
     rootDir,
@@ -779,7 +796,7 @@ async function runCommand(executable, args, options) {
   try {
     const { stdout, stderr } = await execFileAsync(executable, [...args], {
       cwd: options?.cwd,
-      env: process.env
+      env: options?.env ?? process.env
     });
     return { stdout, stderr };
   } catch (error) {
@@ -793,8 +810,17 @@ async function runCommand(executable, args, options) {
     throw withFriendlyGitHubAuthError(error);
   }
 }
+function getGitEnv() {
+  const env = {};
+  for (const [key, value] of Object.entries(process.env)) {
+    if (value !== void 0 && !(key.startsWith("GIT_") && key !== "GIT_SSH_COMMAND")) {
+      env[key] = value;
+    }
+  }
+  return env;
+}
 async function runGit(args, options) {
-  return runCommand("git", args, options);
+  return runCommand("git", args, { ...options, env: getGitEnv() });
 }
 async function runGh(args, options) {
   return runCommand("gh", args, options);
@@ -818,13 +844,11 @@ async function resolveDefaultBranch(repoDir) {
   }
   return "main";
 }
-async function updateCacheRepo(repoDir, baseBranch) {
+async function fetchResultsRepo(repoDir) {
   await runGit(["fetch", "origin", "--prune"], { cwd: repoDir });
-  await runGit(["checkout", baseBranch], { cwd: repoDir });
-  await runGit(["pull", "--ff-only", "origin", baseBranch], { cwd: repoDir });
 }
 function updateStatusFile(config, patch) {
-  const cachePaths = getResultsRepoCachePaths(config.repo);
+  const cachePaths = getResultsRepoLocalPaths(config.repo);
   const current = readPersistedStatus(cachePaths.statusFile);
   writePersistedStatus(cachePaths.statusFile, {
     ...current,
@@ -833,26 +857,31 @@ function updateStatusFile(config, patch) {
 }
 async function ensureResultsRepoClone(config) {
   const normalized = normalizeResultsConfig(config);
-  const cachePaths = getResultsRepoCachePaths(normalized.repo);
+  const cachePaths = getResultsRepoLocalPaths(normalized.repo);
+  const cloneDir = normalized.path;
   mkdirSync(cachePaths.rootDir, { recursive: true });
-  if (!existsSync(cachePaths.repoDir)) {
+  mkdirSync(path4.dirname(cloneDir), { recursive: true });
+  const cloneMissing = !existsSync(cloneDir);
+  const gitDir = path4.join(cloneDir, ".git");
+  const cloneEmpty = !cloneMissing && !existsSync(gitDir) && (await readdir(cloneDir)).length === 0;
+  if (cloneMissing || cloneEmpty) {
     try {
       await runGit([
         "clone",
         "--filter=blob:none",
         resolveResultsRepoUrl(normalized.repo),
-        cachePaths.repoDir
+        cloneDir
       ]);
-      return cachePaths.repoDir;
+      return cloneDir;
     } catch (error) {
       updateStatusFile(normalized, { last_error: withFriendlyGitHubAuthError(error).message });
       throw withFriendlyGitHubAuthError(error);
     }
   }
-  if (!existsSync(path4.join(cachePaths.repoDir, ".git"))) {
-    throw new Error(`Results repo cache is not a git repository: ${cachePaths.repoDir}`);
+  if (!existsSync(gitDir)) {
+    throw new Error(`Results repo clone path is not a git repository: ${cloneDir}`);
   }
-  return cachePaths.repoDir;
+  return cloneDir;
 }
 function getResultsRepoStatus(config) {
   if (!config) {
@@ -860,20 +889,20 @@ function getResultsRepoStatus(config) {
       configured: false,
       available: false,
       repo: "",
-      cache_dir: ""
+      local_dir: ""
     };
   }
   const normalized = normalizeResultsConfig(config);
-  const cachePaths = getResultsRepoCachePaths(normalized.repo);
-  const persisted = readPersistedStatus(cachePaths.statusFile);
+  const localPaths = getResultsRepoLocalPaths(normalized.repo);
+  const persisted = readPersistedStatus(localPaths.statusFile);
   return {
     configured: true,
-    available: existsSync(cachePaths.repoDir),
+    available: existsSync(normalized.path),
     repo: normalized.repo,
     path: normalized.path,
     auto_push: normalized.auto_push,
     branch_prefix: normalized.branch_prefix,
-    cache_dir: cachePaths.repoDir,
+    local_dir: normalized.path,
     last_synced_at: persisted.last_synced_at,
     last_error: persisted.last_error
   };
@@ -882,8 +911,7 @@ async function syncResultsRepo(config) {
   const normalized = normalizeResultsConfig(config);
   try {
     const repoDir = await ensureResultsRepoClone(normalized);
-    const baseBranch = await resolveDefaultBranch(repoDir);
-    await updateCacheRepo(repoDir, baseBranch);
+    await fetchResultsRepo(repoDir);
     updateStatusFile(normalized, {
       last_synced_at: (/* @__PURE__ */ new Date()).toISOString(),
       last_error: void 0
@@ -900,7 +928,7 @@ async function checkoutResultsRepoBranch(config, branchName) {
   const normalized = normalizeResultsConfig(config);
   const repoDir = await ensureResultsRepoClone(normalized);
   const baseBranch = await resolveDefaultBranch(repoDir);
-  await updateCacheRepo(repoDir, baseBranch);
+  await fetchResultsRepo(repoDir);
   await runGit(["checkout", "-B", branchName, `origin/${baseBranch}`], { cwd: repoDir });
   updateStatusFile(normalized, { last_error: void 0 });
   return {
@@ -913,7 +941,7 @@ async function prepareResultsRepoBranch(config, branchName) {
   const normalized = normalizeResultsConfig(config);
   const cloneDir = await ensureResultsRepoClone(normalized);
   const baseBranch = await resolveDefaultBranch(cloneDir);
-  await updateCacheRepo(cloneDir, baseBranch);
+  await fetchResultsRepo(cloneDir);
   const worktreeRoot = await mkdtemp(path4.join(os.tmpdir(), "agentv-results-repo-"));
   const worktreeDir = path4.join(worktreeRoot, "repo");
   await runGit(["worktree", "add", "-B", branchName, worktreeDir, `origin/${baseBranch}`], {
@@ -939,10 +967,7 @@ async function stageResultsArtifacts(params) {
 }
 function resolveResultsRepoRunsDir(config) {
   const normalized = normalizeResultsConfig(config);
-  return path4.join(
-    getResultsRepoCachePaths(normalized.repo).repoDir,
-    ...normalized.path.split("/")
-  );
+  return path4.join(normalized.path, "runs");
 }
 async function directorySizeBytes(targetPath) {
   const entry = await stat(targetPath);
@@ -971,7 +996,7 @@ async function commitAndPushResultsBranch(params) {
 async function pushResultsRepoBranch(config, branchName, cwd) {
   const normalized = normalizeResultsConfig(config);
   await runGit(["push", "-u", "origin", branchName], {
-    cwd: cwd ?? getResultsRepoCachePaths(normalized.repo).repoDir
+    cwd: cwd ?? normalized.path
   });
   updateStatusFile(normalized, {
     last_synced_at: (/* @__PURE__ */ new Date()).toISOString(),
@@ -1004,8 +1029,8 @@ async function directPushResults(params) {
   const normalized = normalizeResultsConfig(params.config);
   const repoDir = await ensureResultsRepoClone(normalized);
   const baseBranch = await resolveDefaultBranch(repoDir);
-  await updateCacheRepo(repoDir, baseBranch);
-  const destinationDir = path4.join(repoDir, normalized.path, params.destinationPath);
+  await fetchResultsRepo(repoDir);
+  const destinationDir = path4.join(repoDir, "runs", params.destinationPath);
   await stageResultsArtifacts({
     repoDir,
     sourceDir: params.sourceDir,
@@ -1019,10 +1044,19 @@ async function directPushResults(params) {
   if (status.trim().length === 0) {
     return false;
   }
-  await runGit(["commit", "-m", params.commitMessage], { cwd: repoDir });
+  await runGit(
+    [
+      "commit",
+      "-m",
+      params.commitMessage,
+      "-m",
+      `Agentv-Run: ${buildGitRunId(params.destinationPath)}`
+    ],
+    { cwd: repoDir }
+  );
   for (let attempt = 1; attempt <= DIRECT_PUSH_MAX_RETRIES; attempt++) {
     try {
-      await runGit(["push", "origin", baseBranch], { cwd: repoDir });
+      await runGit(["push", "origin", `HEAD:${baseBranch}`], { cwd: repoDir });
       updateStatusFile(normalized, {
         last_synced_at: (/* @__PURE__ */ new Date()).toISOString(),
         last_error: void 0
@@ -1031,7 +1065,8 @@ async function directPushResults(params) {
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       if (attempt < DIRECT_PUSH_MAX_RETRIES && message.includes("non-fast-forward")) {
-        await runGit(["pull", "--rebase", "origin", baseBranch], { cwd: repoDir });
+        await fetchResultsRepo(repoDir);
+        await runGit(["rebase", `origin/${baseBranch}`], { cwd: repoDir });
       } else {
         throw error;
       }
@@ -1039,6 +1074,195 @@ async function directPushResults(params) {
   }
   return false;
 }
+function buildGitRunId(relativeRunPath) {
+  const normalized = relativeRunPath.split(path4.sep).join("/");
+  const segments = normalized.split("/").filter(Boolean);
+  if (segments.length >= 2) {
+    const experiment = segments.slice(0, -1).join("/");
+    const timestamp = segments.at(-1);
+    if (experiment === "default") {
+      return timestamp ?? normalized;
+    }
+    return `${experiment}::${timestamp}`;
+  }
+  return segments[0] ?? relativeRunPath;
+}
+function getRunExperiment(runId, benchmark) {
+  const experiment = benchmark.metadata?.experiment?.trim();
+  if (experiment) {
+    return experiment;
+  }
+  const separatorIndex = runId.lastIndexOf("::");
+  return separatorIndex === -1 ? "default" : runId.slice(0, separatorIndex);
+}
+function computeAveragePassRate(runSummary) {
+  if (!runSummary) {
+    return void 0;
+  }
+  const passRates = Object.values(runSummary).map((summary) => summary.pass_rate?.mean).filter((value) => typeof value === "number" && Number.isFinite(value));
+  if (passRates.length === 0) {
+    return void 0;
+  }
+  return passRates.reduce((sum, value) => sum + value, 0) / passRates.length;
+}
+async function runGitBatch(repoDir, input) {
+  return new Promise((resolve, reject) => {
+    const child = spawn("git", ["cat-file", "--batch"], {
+      cwd: repoDir,
+      env: getGitEnv(),
+      stdio: ["pipe", "pipe", "pipe"]
+    });
+    const stdoutChunks = [];
+    const stderrChunks = [];
+    child.stdout.on("data", (chunk) => {
+      stdoutChunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+    });
+    child.stderr.on("data", (chunk) => {
+      stderrChunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+    });
+    child.on("error", (error) => reject(withFriendlyGitHubAuthError(error)));
+    child.on("close", (code) => {
+      if (code === 0) {
+        resolve(Buffer.concat(stdoutChunks));
+        return;
+      }
+      const stderr = Buffer.concat(stderrChunks).toString("utf8").trim();
+      reject(
+        withFriendlyGitHubAuthError(
+          stderr.length > 0 ? new Error(stderr) : new Error("git cat-file failed")
+        )
+      );
+    });
+    child.stdin.end(input);
+  });
+}
+function parseGitBatchBlobs(output) {
+  const blobs = [];
+  let offset = 0;
+  while (offset < output.length) {
+    const headerEnd = output.indexOf(10, offset);
+    if (headerEnd === -1) {
+      throw new Error("Malformed git cat-file output: missing header terminator");
+    }
+    const header = output.subarray(offset, headerEnd).toString("utf8");
+    offset = headerEnd + 1;
+    if (header.length === 0) {
+      continue;
+    }
+    const missingMatch = /^(.*) missing$/.exec(header);
+    if (missingMatch) {
+      continue;
+    }
+    const headerMatch = /^(.*) (\w+) (\d+)$/.exec(header);
+    if (!headerMatch) {
+      throw new Error(`Malformed git cat-file header: ${header}`);
+    }
+    const [, objectRef, objectType, sizeText] = headerMatch;
+    if (objectType !== "blob") {
+      throw new Error(`Unsupported git object type for ${objectRef}: ${objectType}`);
+    }
+    const size = Number.parseInt(sizeText, 10);
+    const contentEnd = offset + size;
+    if (contentEnd > output.length) {
+      throw new Error(`Malformed git cat-file output for ${objectRef}: truncated blob content`);
+    }
+    blobs.push({
+      size,
+      content: output.subarray(offset, contentEnd)
+    });
+    offset = contentEnd;
+    if (offset < output.length && output[offset] === 10) {
+      offset += 1;
+    }
+  }
+  return blobs;
+}
+async function listGitRuns(repoDir, ref = "origin/main") {
+  const { stdout: treeOut } = await runGit(["ls-tree", "-r", "--name-only", ref, "runs"], {
+    cwd: repoDir
+  });
+  const benchmarkPaths = treeOut.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.endsWith("/benchmark.json"));
+  if (benchmarkPaths.length === 0) {
+    return [];
+  }
+  const batchInput = `${benchmarkPaths.map((benchmarkPath) => `${ref}:${benchmarkPath}`).join("\n")}
+`;
+  const blobs = parseGitBatchBlobs(await runGitBatch(repoDir, batchInput));
+  if (blobs.length !== benchmarkPaths.length) {
+    throw new Error(
+      `Expected ${benchmarkPaths.length} git blobs but received ${blobs.length} while listing results runs`
+    );
+  }
+  const runs = blobs.flatMap((blob, index) => {
+    const benchmarkPath = benchmarkPaths[index];
+    const benchmark = JSON.parse(blob.content.toString("utf8"));
+    const runDir = path4.posix.dirname(benchmarkPath);
+    const relativeRunPath = path4.posix.relative("runs", runDir);
+    const runId = buildGitRunId(relativeRunPath);
+    const timestamp = benchmark.metadata?.timestamp?.trim() || path4.posix.basename(runDir);
+    const targets = benchmark.metadata?.targets ?? [];
+    const passRate = computeAveragePassRate(benchmark.run_summary);
+    return [
+      {
+        run_id: runId,
+        experiment: getRunExperiment(runId, benchmark),
+        timestamp,
+        ...passRate !== void 0 && { pass_rate: passRate },
+        ...targets.length === 1 && targets[0] ? { target: targets[0] } : {},
+        manifest_path: path4.posix.join(runDir, "index.jsonl"),
+        benchmark_path: benchmarkPath,
+        display_name: path4.posix.basename(runDir),
+        test_count: benchmark.metadata?.tests_run?.length ?? 0,
+        avg_score: 0,
+        size_bytes: blob.size
+      }
+    ];
+  });
+  runs.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
+  return runs;
+}
+async function materializeGitRun(repoDir, relativeRunPath, ref = "origin/main") {
+  const normalizedRunPath = relativeRunPath.split(path4.sep).join("/");
+  const runTreePath = path4.posix.join("runs", normalizedRunPath);
+  const targetRunDir = path4.join(repoDir, ...runTreePath.split("/"));
+  const { stdout: treeOut } = await runGit(["ls-tree", "-r", "--name-only", ref, runTreePath], {
+    cwd: repoDir
+  });
+  const filePaths = treeOut.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
+  if (filePaths.length === 0) {
+    return;
+  }
+  const batchInput = `${filePaths.map((filePath) => `${ref}:${filePath}`).join("\n")}
+`;
+  const blobs = parseGitBatchBlobs(await runGitBatch(repoDir, batchInput));
+  if (blobs.length !== filePaths.length) {
+    throw new Error(
+      `Expected ${filePaths.length} git blobs but received ${blobs.length} while materializing results run`
+    );
+  }
+  const tempRoot = mkdtempSync(path4.join(repoDir, ".agentv-run-"));
+  const tempRunDir = path4.join(tempRoot, "run");
+  try {
+    for (const [index, filePath] of filePaths.entries()) {
+      const relativeFilePath = path4.posix.relative(runTreePath, filePath);
+      const absolutePath = path4.join(tempRunDir, ...relativeFilePath.split("/"));
+      mkdirSync(path4.dirname(absolutePath), { recursive: true });
+      writeFileSync(absolutePath, blobs[index].content);
+    }
+    mkdirSync(path4.dirname(targetRunDir), { recursive: true });
+    try {
+      renameSync(tempRunDir, targetRunDir);
+    } catch (error) {
+      const code = typeof error === "object" && error !== null && "code" in error ? error.code : void 0;
+      if ((code === "EEXIST" || code === "ENOTEMPTY") && existsSync(targetRunDir)) {
+        return;
+      }
+      throw error;
+    }
+  } finally {
+    rmSync(tempRoot, { recursive: true, force: true });
+  }
+}
 
 // src/projects.ts
 import {
@@ -1046,7 +1270,7 @@ import {
   mkdirSync as mkdirSync2,
   readFileSync as readFileSync3,
   readdirSync,
-  renameSync,
+  renameSync as renameSync2,
   statSync,
   unlinkSync,
   writeFileSync as writeFileSync2
@@ -1087,7 +1311,7 @@ function migrateLegacyBenchmarksFile() {
   try {
     mkdirSync2(path5.dirname(newPath), { recursive: true });
     writeFileSync2(tempPath, newContent, "utf-8");
-    renameSync(tempPath, newPath);
+    renameSync2(tempPath, newPath);
     unlinkSync(oldPath);
   } catch (err) {
     try {
@@ -2639,7 +2863,7 @@ export {
   getOutputFilenames,
   getProject,
   getProjectsRegistryPath,
-  getResultsRepoCachePaths,
+  getResultsRepoLocalPaths,
   getResultsRepoStatus,
   getSubagentsRoot,
   getTextContent,
@@ -2659,6 +2883,7 @@ export {
   isTestMessage,
   isTestMessageRole,
   killAllTrackedChildren,
+  listGitRuns,
   listTargetNames,
   loadConfig,
   loadEvalCaseById,
@@ -2670,6 +2895,7 @@ export {
   loadTests,
   loadTsConfig,
   loadTsEvalFile,
+  materializeGitRun,
   mergeExecutionMetrics,
   negateScore,
   normalizeLineEndings,