@forwardimpact/libeval 0.1.50 → 0.1.52

package/src/trace-github.js

@@ -1,10 +1,9 @@
-import { execSync } from "node:child_process";
-import { createWriteStream } from "node:fs";
-import { mkdir } from "node:fs/promises";
 import path from "node:path";
 import { pipeline } from "node:stream/promises";
 import { Readable } from "node:stream";
 
+import { isoTimestamp } from "@forwardimpact/libutil";
+
 const API = "https://api.github.com";
 
 /**
@@ -17,11 +16,15 @@ export class TraceGitHub {
    * @param {string} deps.token - GitHub token
    * @param {string} deps.owner - Repository owner
    * @param {string} deps.repo  - Repository name
+   * @param {import("@forwardimpact/libutil/runtime").Runtime} deps.runtime -
+   *   Ambient collaborators; uses `fs`, `subprocess`, `clock`.
    */
-  constructor({ token, owner, repo }) {
+  constructor({ token, owner, repo, runtime }) {
+    if (!runtime) throw new Error("runtime is required");
     this.token = token;
     this.owner = owner;
     this.repo = repo;
+    this.runtime = runtime;
   }
 
   /**
@@ -35,7 +38,7 @@ export class TraceGitHub {
    */
   async listRuns(opts = {}) {
     const { pattern = "agent", limit = 50, lookback = "7d" } = opts;
-    const cutoff = parseLookback(lookback);
+    const cutoff = parseLookback(lookback, this.runtime.clock.now());
 
     const params = new URLSearchParams({
       per_page: String(Math.min(limit, 100)),
@@ -77,8 +80,9 @@ export class TraceGitHub {
    * @returns {Promise<{dir: string, artifact: string, files: string[]}>}
    */
   async downloadTrace(runId, opts = {}) {
+    const fs = this.runtime.fs;
     const dir = opts.dir ?? `/tmp/trace-${runId}`;
-    await mkdir(dir, { recursive: true });
+    await fs.mkdir(dir, { recursive: true });
 
     // List artifacts for this run.
     const url = `${API}/repos/${this.owner}/${this.repo}/actions/runs/${runId}/artifacts`;
@@ -121,15 +125,27 @@ export class TraceGitHub {
     }
 
     // Stream to disk then extract.
-    await pipeline(Readable.fromWeb(response.body), createWriteStream(zipPath));
-
-    execSync(
-      `unzip -o -q ${JSON.stringify(zipPath)} -d ${JSON.stringify(dir)}`,
+    await pipeline(
+      Readable.fromWeb(response.body),
+      fs.createWriteStream(zipPath),
     );
 
+    const unzip = await this.runtime.subprocess.run("unzip", [
+      "-o",
+      "-q",
+      zipPath,
+      "-d",
+      dir,
+    ]);
+    if (unzip.exitCode !== 0) {
+      throw new Error(
+        `unzip failed (${unzip.exitCode}): ${unzip.stderr || unzip.stdout}`,
+      );
+    }
+
     // List extracted files.
-    const { readdirSync } = await import("node:fs");
-    const files = readdirSync(dir).filter((f) => !f.endsWith(".zip"));
+    const entries = await fs.readdir(dir);
+    const files = entries.filter((f) => !f.endsWith(".zip"));
 
     return { dir, artifact: artifact.name, files };
   }
@@ -160,14 +176,15 @@ export class TraceGitHub {
  * Parse a lookback duration string into an ISO date string.
  * Supports: Nd (days), Nh (hours), Nw (weeks).
  * @param {string} lookback
+ * @param {number} nowMs - Current time in ms (`runtime.clock.now()`).
  * @returns {string|null} ISO date string or null if unparseable
  */
-function parseLookback(lookback) {
+function parseLookback(lookback, nowMs) {
   const match = lookback.match(/^(\d+)([dhw])$/);
   if (!match) return null;
   const [, val, unit] = match;
   const ms = { d: 86400000, h: 3600000, w: 604800000 }[unit];
-  return new Date(Date.now() - parseInt(val, 10) * ms).toISOString();
+  return isoTimestamp(nowMs - parseInt(val, 10) * ms);
 }
 
 /**
@@ -203,22 +220,23 @@ export function parseGitRemote(remote) {
  *   1. `GITHUB_REPOSITORY` env var (set automatically by GitHub Actions).
  *   2. `git remote get-url origin` in the current working directory.
  *
- * @returns {{owner: string, repo: string}}
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @returns {Promise<{owner: string, repo: string}>}
  * @throws {Error} with a clear message if neither source yields a parseable slug.
  */
-export function detectRepoSlug() {
-  const env = process.env.GITHUB_REPOSITORY;
+export async function detectRepoSlug(runtime) {
+  const env = runtime.proc.env.GITHUB_REPOSITORY;
   if (env && env.trim()) {
     return parseGitRemote(env.trim());
   }
 
-  let remote;
-  try {
-    remote = execSync("git remote get-url origin", {
-      encoding: "utf8",
-      stdio: ["ignore", "pipe", "ignore"],
-    }).trim();
-  } catch {
+  const result = await runtime.subprocess.run("git", [
+    "remote",
+    "get-url",
+    "origin",
+  ]);
+  const remote = result.exitCode === 0 ? result.stdout.trim() : "";
+  if (result.exitCode !== 0) {
     throw new Error(
       "Cannot detect repository: set --repo <owner/repo>, export GITHUB_REPOSITORY, or run inside a git checkout with an 'origin' remote.",
     );
@@ -245,10 +263,12 @@ export function detectRepoSlug() {
  * @param {object} opts
  * @param {string} opts.token - GitHub token (e.g. from `Config.ghToken()`)
  * @param {string} [opts.repo] - "owner/repo" override (default: detect from git remote)
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} opts.runtime - Ambient collaborators.
  * @returns {Promise<TraceGitHub>}
  */
 export async function createTraceGitHub(opts = {}) {
-  const { token, repo: repoOverride } = opts;
+  const { token, repo: repoOverride, runtime } = opts;
+  if (!runtime) throw new Error("createTraceGitHub: runtime is required");
   if (!token) {
     throw new Error(
       "createTraceGitHub: token is required (pass Config.ghToken())",
@@ -257,7 +277,7 @@ export async function createTraceGitHub(opts = {}) {
 
   const { owner, repo } = repoOverride
     ? parseGitRemote(repoOverride)
-    : detectRepoSlug();
+    : await detectRepoSlug(runtime);
 
-  return new TraceGitHub({ token, owner, repo });
+  return new TraceGitHub({ token, owner, repo, runtime });
 }

package/src/benchmark/scorer.js

@@ -1,138 +0,0 @@
-/**
- * Scorer — runs `<task.paths.hooks>/score.sh` from the template path against
- * the post-run agent CWD. The exit code is authoritative for the verdict;
- * structured per-test rows arrive on fd 3 (`$RESULTS_FD=3`) as NDJSON.
- */
-
-import { spawn } from "node:child_process";
-import {
-  closeSync,
-  createWriteStream,
-  openSync,
-  readFileSync,
-  unlinkSync,
-} from "node:fs";
-import { join } from "node:path";
-
-/**
- * @typedef {object} ScoringResult
- * @property {"pass" | "fail"} verdict
- * @property {Array<object>} details
- * @property {number} exitCode
- */
-
-/**
- * Run the task's scoring script.
- * @param {import("./task-family.js").Task} task
- * @param {{cwd: string, port: number, runDir: string}} ctx
- * @returns {Promise<ScoringResult>}
- */
-export function runScoring(task, ctx) {
-  if (!task.paths.score) {
-    return Promise.resolve({ verdict: "pass", details: [], exitCode: 0 });
-  }
-  return new Promise((res, rej) => {
-    const script = task.paths.score;
-    const stderrLog = createWriteStream(join(ctx.runDir, "scoring.stderr.log"));
-
-    // Bun's child_process pipe setup for fd >= 3 is racy under load (it
-    // creates a unix socket pair and the connect() can return ENOENT). Use
-    // a temp file as the fd-3 backing store instead — the script still
-    // writes via `$RESULTS_FD`, but we hand it a real file descriptor.
-    const fd3Path = join(ctx.runDir, "scoring.fd3.ndjson");
-    let fd3File;
-    try {
-      fd3File = openSync(fd3Path, "w+");
-    } catch (e) {
-      rej(e);
-      return;
-    }
-
-    const child = spawn(script, [], {
-      env: {
-        ...process.env,
-        WORKDIR: ctx.cwd,
-        PORT: String(ctx.port),
-        RESULTS_FD: "3",
-      },
-      stdio: ["inherit", "pipe", "pipe", fd3File],
-    });
-    if (child.pid === undefined) {
-      try {
-        closeSync(fd3File);
-      } catch {
-        // already closed
-      }
-      rej(new Error(`failed to spawn scoring script: ${script}`));
-      return;
-    }
-
-    child.stderr.pipe(stderrLog);
-    // Drain stdout (do not require consumers to read it).
-    child.stdout.on("data", () => {});
-
-    child.on("error", (e) => {
-      tryClose(fd3File);
-      rej(e);
-    });
-    child.on("close", (code) => {
-      stderrLog.end();
-      tryClose(fd3File);
-      const raw = readAndUnlink(fd3Path);
-      const details = [];
-      parseFd3Buffer(raw, details);
-      const exitCode = typeof code === "number" ? code : -1;
-      res({
-        verdict: exitCode === 0 ? "pass" : "fail",
-        details,
-        exitCode,
-      });
-    });
-  });
-}
-
-function pushRow(line, details) {
-  const trimmed = line.trim();
-  if (!trimmed) return;
-  try {
-    details.push(JSON.parse(trimmed));
-  } catch {
-    details.push({ raw: trimmed, parseError: true });
-  }
-}
-
-function tryClose(fd) {
-  try {
-    closeSync(fd);
-  } catch {
-    // already closed
-  }
-}
-
-function readAndUnlink(path) {
-  let raw = "";
-  try {
-    raw = readFileSync(path, "utf8");
-  } catch {
-    // empty
-  }
-  try {
-    unlinkSync(path);
-  } catch {
-    // best-effort cleanup
-  }
-  return raw;
-}
-
-/**
- * Parse the fd-3 buffer (read from the temp-file backing) into one NDJSON
- * row per detail entry.
- */
-function parseFd3Buffer(buf, details) {
-  if (!buf) return;
-  const parts = buf.split("\n");
-  for (let i = 0; i < parts.length - 1; i++) pushRow(parts[i], details);
-  if (parts[parts.length - 1].trim()) {
-    pushRow(parts[parts.length - 1], details);
-  }
-}

package/src/commands/benchmark-score.js

@@ -1,68 +0,0 @@
-/**
- * `fit-benchmark score` — score a single task against a post-run workdir
- * directory without invoking an agent (P6/P7). Useful for re-scoring an
- * agent's output against revised grading material.
- */
-
-import { writeFileSync } from "node:fs";
-import { join, resolve } from "node:path";
-import { createServer } from "node:net";
-
-import { validateScoringRecord } from "../benchmark/result.js";
-import { runScoring } from "../benchmark/scorer.js";
-import { loadTaskFamily } from "../benchmark/task-family.js";
-
-/**
- * @param {object} values
- * @param {string[]} _args
- */
-export async function runBenchmarkScoreCommand(values, _args) {
-  const familyInput = values.family;
-  if (!familyInput) throw new Error("--family is required");
-  const taskId = values.task;
-  if (!taskId) throw new Error("--task is required");
-  const workdirArg = values.workdir;
-  if (!workdirArg) throw new Error("--workdir is required");
-
-  const family = await loadTaskFamily(familyInput);
-  const task = family.tasks().find((t) => t.id === taskId);
-  if (!task) throw new Error(`task not found in family: ${taskId}`);
-
-  const runDir = resolve(workdirArg);
-  const cwd = join(runDir, "cwd");
-  const port = await allocatePort();
-
-  const scoring = await runScoring(task, { cwd, port, runDir });
-  const record = {
-    taskId: task.id,
-    scoring,
-    exitCode: scoring.exitCode,
-  };
-  validateScoringRecord(record);
-
-  const line = JSON.stringify(record) + "\n";
-  if (values.output) {
-    writeFileSync(resolve(values.output), line);
-  } else {
-    process.stdout.write(line);
-  }
-  process.exit(scoring.verdict === "pass" ? 0 : 1);
-}
-
-function allocatePort() {
-  return new Promise((res, rej) => {
-    const server = createServer();
-    server.unref();
-    server.on("error", rej);
-    server.listen(0, "127.0.0.1", () => {
-      const addr = server.address();
-      if (!addr || typeof addr === "string") {
-        server.close();
-        rej(new Error("failed to allocate port"));
-        return;
-      }
-      const port = addr.port;
-      server.close(() => res(port));
-    });
-  });
-}