@zhijiewang/openharness 2.38.0 → 2.40.0

package/dist/evals/orchestrator.js

@@ -0,0 +1,391 @@
+/**
+ * oh evals — run orchestrator.
+ *
+ * Coordinates the full run lifecycle:
+ *   - manages a concurrency pool of N parallel task workers
+ *   - per task: extract repo tarball → setup.sh → spawn `oh run` subprocess
+ *     → tee stdout to transcript file + parse stream-json → git diff →
+ *     scoreTask → RunWriter.appendResult → cleanup worktree
+ *   - aggregates total cost; halts scheduling when total >= max_cost_usd
+ *   - resumability: skip instance_ids already in results.jsonl
+ *   - cancellation: cancel() sets flag, SIGTERMs running subs, then SIGKILL
+ *
+ * Subprocess command (no --working-dir flag — we use spawn's cwd option):
+ *   node dist/main.js run --bare --output-format stream-json
+ *     --no-session-persistence --max-budget-usd <cap> --max-turns <n>
+ *     --model <model> "<problem_statement>"
+ */
+import { execFileSync, spawn, spawnSync } from "node:child_process";
+import { createWriteStream, existsSync, mkdirSync, rmSync as nodeRmSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { isGitRepo, removeWorktree } from "../git/index.js";
+import { RunWriter } from "./run-writer.js";
+import { scoreTask } from "./scorer.js";
+export class RunOrchestrator {
+    opts;
+    writer;
+    perTaskCap;
+    cancelled = false;
+    halted = false;
+    totalCost = 0;
+    running = new Set();
+    skipIds = new Set();
+    constructor(opts) {
+        this.opts = opts;
+        this.perTaskCap = opts.maxTaskCostUsd ?? opts.maxCostUsd / Math.max(1, opts.tasks.length);
+        const harnessVersion = readHarnessVersion();
+        this.writer = new RunWriter(opts.runDir, {
+            run_id: pathBaseName(opts.runDir),
+            pack: opts.pack.name,
+            pack_version: opts.pack.version,
+            model: opts.model,
+            harness_version: harnessVersion,
+            max_cost_usd: opts.maxCostUsd,
+            started_at: new Date().toISOString(),
+        });
+        if (opts.resumeFromRunId) {
+            const prior = this.writer.loadExistingResults();
+            for (const r of prior) {
+                this.skipIds.add(r.instance_id);
+                this.totalCost += r.cost_usd;
+            }
+        }
+    }
+    cancel() {
+        this.cancelled = true;
+        for (const child of this.running) {
+            try {
+                child.kill("SIGTERM");
+            }
+            catch {
+                // already exited
+            }
+        }
+    }
+    async run() {
+        const queue = this.opts.tasks.filter((t) => !this.skipIds.has(t.instance_id));
+        let nextIndex = 0;
+        const concurrency = Math.max(1, this.opts.concurrency);
+        const worker = async () => {
+            while (!this.cancelled && !this.halted) {
+                if (this.totalCost >= this.opts.maxCostUsd) {
+                    this.halted = true;
+                    break;
+                }
+                const idx = nextIndex++;
+                if (idx >= queue.length)
+                    break;
+                const task = queue[idx];
+                this.opts.onTaskStart?.(task);
+                const result = await this.runOneTask(task);
+                this.totalCost += result.cost_usd;
+                this.writer.appendResult(result);
+                this.opts.onTaskComplete?.(result);
+            }
+        };
+        await Promise.all(Array.from({ length: concurrency }, worker));
+        return this.writer.finalize({
+            partial: this.cancelled || this.halted,
+            finished_at: new Date().toISOString(),
+        });
+    }
+    async runOneTask(task) {
+        const startedAt = new Date();
+        const start = Date.now();
+        const taskWorktreeBase = join(this.opts.runDir, "worktrees", task.instance_id);
+        mkdirSync(taskWorktreeBase, { recursive: true });
+        let worktreePath = null;
+        let usedGitWorktree = false;
+        try {
+            // 1. Extract fixture into the per-task worktree dir.
+            worktreePath = taskWorktreeBase;
+            await extractFixture(this.opts.packDir, task.instance_id, worktreePath);
+            // 2. Run setup.sh (creates a base commit so we can git diff later).
+            const setupOk = await runSetupScript(this.opts.packDir, task.instance_id, worktreePath);
+            if (!setupOk.ok) {
+                return makeResult({
+                    task,
+                    status: "skipped",
+                    resolved: false,
+                    cost_usd: 0,
+                    turns_used: 0,
+                    duration_ms: Date.now() - start,
+                    model_patch: "",
+                    tests_status: emptyTestsStatus(),
+                    transcript_path: `transcripts/${task.instance_id}.jsonl`,
+                    error_message: `setup.sh failed: ${setupOk.error}`,
+                    startedAt,
+                });
+            }
+            usedGitWorktree = isGitRepo(worktreePath);
+            // 3. Spawn the subprocess.
+            const { exec, args } = this.opts.subprocessArgvBuilder
+                ? this.opts.subprocessArgvBuilder(task, {
+                    worktreeDir: worktreePath,
+                    perTaskCostCap: this.perTaskCap,
+                    maxTurns: this.opts.maxTaskTurns,
+                    model: this.opts.model,
+                })
+                : {
+                    exec: process.execPath,
+                    args: defaultRunArgs({
+                        ohEntry: this.opts.ohEntry ?? defaultOhEntry(),
+                        perTaskCostCap: this.perTaskCap,
+                        maxTurns: this.opts.maxTaskTurns,
+                        model: this.opts.model,
+                        fallbackModel: this.opts.fallbackModel,
+                        prompt: task.problem_statement,
+                    }),
+                };
+            const transcriptPath = join(this.opts.runDir, "transcripts", `${task.instance_id}.jsonl`);
+            const transcriptStream = createWriteStream(transcriptPath);
+            const child = spawn(exec, args, {
+                cwd: worktreePath,
+                env: { ...process.env },
+                stdio: ["ignore", "pipe", "pipe"],
+            });
+            this.running.add(child);
+            // Tee stdout to transcript file + parser.
+            let stdoutBuf = "";
+            child.stdout?.on("data", (chunk) => {
+                transcriptStream.write(chunk);
+                stdoutBuf += chunk.toString("utf-8");
+            });
+            let stderrBuf = "";
+            child.stderr?.on("data", (chunk) => {
+                stderrBuf += chunk.toString("utf-8");
+            });
+            // 4. Race subprocess vs timeout.
+            let timedOut = false;
+            const timeoutHandle = setTimeout(() => {
+                timedOut = true;
+                try {
+                    child.kill("SIGKILL");
+                }
+                catch {
+                    /* already exited */
+                }
+            }, this.opts.taskTimeoutMs);
+            const exitCode = await new Promise((resolve) => {
+                child.once("exit", (code) => resolve(code));
+                child.once("error", () => resolve(null));
+            });
+            clearTimeout(timeoutHandle);
+            this.running.delete(child);
+            // Flush the transcript stream before proceeding to cleanup. On Windows,
+            // unflushed write streams can keep file handles open, which races with
+            // worktree rmSync in the finally block.
+            await new Promise((resolve) => {
+                transcriptStream.end(() => resolve());
+            });
+            // 5. Parse stream-json result event.
+            const parsed = parseStreamJsonResult(stdoutBuf);
+            if (timedOut) {
+                return makeResult({
+                    task,
+                    status: "timeout",
+                    resolved: false,
+                    cost_usd: parsed.cost_usd,
+                    turns_used: parsed.turns_used,
+                    duration_ms: Date.now() - start,
+                    model_patch: usedGitWorktree ? captureGitDiff(worktreePath) : "",
+                    tests_status: emptyTestsStatus(),
+                    transcript_path: `transcripts/${task.instance_id}.jsonl`,
+                    error_message: `task exceeded ${this.opts.taskTimeoutMs}ms timeout`,
+                    startedAt,
+                });
+            }
+            if (parsed.exit_reason === "budget_exceeded") {
+                return makeResult({
+                    task,
+                    status: "budget_exceeded",
+                    resolved: false,
+                    cost_usd: parsed.cost_usd,
+                    turns_used: parsed.turns_used,
+                    duration_ms: Date.now() - start,
+                    model_patch: usedGitWorktree ? captureGitDiff(worktreePath) : "",
+                    tests_status: emptyTestsStatus(),
+                    transcript_path: `transcripts/${task.instance_id}.jsonl`,
+                    startedAt,
+                });
+            }
+            if (exitCode !== 0) {
+                return makeResult({
+                    task,
+                    status: "error",
+                    resolved: false,
+                    cost_usd: parsed.cost_usd,
+                    turns_used: parsed.turns_used,
+                    duration_ms: Date.now() - start,
+                    model_patch: usedGitWorktree ? captureGitDiff(worktreePath) : "",
+                    tests_status: emptyTestsStatus(),
+                    transcript_path: `transcripts/${task.instance_id}.jsonl`,
+                    error_message: `subprocess exit ${exitCode}: ${stderrBuf.slice(-500)}`,
+                    startedAt,
+                });
+            }
+            // 6. Capture model_patch.
+            const modelPatch = usedGitWorktree ? captureGitDiff(worktreePath) : "";
+            // 7. Score.
+            const score = await scoreTask({
+                task,
+                worktreeDir: worktreePath,
+                fixtureDir: join(this.opts.packDir, "fixtures", task.instance_id),
+                packDefaultTestCommand: this.opts.pack.default_test_command,
+                testTimeoutMs: this.opts.taskTimeoutMs,
+            });
+            const status = score.error_message !== undefined ? "error" : score.resolved ? "resolved" : "failed";
+            return makeResult({
+                task,
+                status,
+                resolved: score.resolved,
+                cost_usd: parsed.cost_usd,
+                turns_used: parsed.turns_used,
+                duration_ms: Date.now() - start,
+                model_patch: modelPatch,
+                tests_status: score.tests_status,
+                transcript_path: `transcripts/${task.instance_id}.jsonl`,
+                error_message: score.error_message,
+                startedAt,
+            });
+        }
+        finally {
+            // Clean up worktree (best-effort; swallow errors so a leak doesn't stop a run).
+            if (worktreePath && existsSync(worktreePath)) {
+                try {
+                    if (usedGitWorktree)
+                        removeWorktree(worktreePath);
+                    // Also remove the temp dir tree under runDir/worktrees/<id> regardless.
+                    rmSyncIfExists(worktreePath);
+                }
+                catch {
+                    /* swallow */
+                }
+            }
+        }
+    }
+}
+// ── helpers ──
+function rmSyncIfExists(p) {
+    try {
+        nodeRmSync(p, { recursive: true, force: true });
+    }
+    catch {
+        /* swallow */
+    }
+}
+function emptyTestsStatus() {
+    return {
+        FAIL_TO_PASS: { success: [], failure: [] },
+        PASS_TO_PASS: { success: [], failure: [] },
+    };
+}
+function makeResult(args) {
+    return {
+        instance_id: args.task.instance_id,
+        status: args.status,
+        resolved: args.resolved,
+        cost_usd: args.cost_usd,
+        turns_used: args.turns_used,
+        duration_ms: args.duration_ms,
+        model_patch: args.model_patch,
+        tests_status: args.tests_status,
+        transcript_path: args.transcript_path,
+        error_message: args.error_message,
+        started_at: args.startedAt.toISOString(),
+        finished_at: new Date().toISOString(),
+    };
+}
+function parseStreamJsonResult(stdout) {
+    const lines = stdout.split("\n").filter((l) => l.trim().length > 0);
+    for (let i = lines.length - 1; i >= 0; i--) {
+        try {
+            const evt = JSON.parse(lines[i]);
+            if (evt.type === "result") {
+                return {
+                    cost_usd: Number(evt.total_cost_usd ?? 0),
+                    turns_used: Number(evt.num_turns ?? 0),
+                    exit_reason: String(evt.subtype ?? "ok"),
+                    final_message: String(evt.result ?? ""),
+                };
+            }
+        }
+        catch {
+            /* not JSON; skip */
+        }
+    }
+    return { cost_usd: 0, turns_used: 0, exit_reason: "ok", final_message: "" };
+}
+function captureGitDiff(worktreeDir) {
+    try {
+        return execFileSync("git", ["-C", worktreeDir, "diff", "HEAD"], { encoding: "utf-8" });
+    }
+    catch {
+        return "";
+    }
+}
+async function extractFixture(packDir, instanceId, dest) {
+    const tarPath = join(packDir, "fixtures", instanceId, "repo.tar.zst");
+    if (!existsSync(tarPath) || readFileSync(tarPath).length === 0) {
+        // Empty tarball = test mode (synthetic pack). Caller's setup.sh
+        // handles initialization; we just ensure the dest dir exists.
+        return;
+    }
+    // Extract via system `tar` (Linux/macOS/Win10+ all ship one).
+    // -I/--use-compress-program zstd is available on tar 1.31+.
+    execFileSync("tar", ["--use-compress-program=zstd -d", "-xf", tarPath, "-C", dest], {
+        stdio: ["ignore", "pipe", "pipe"],
+    });
+}
+async function runSetupScript(packDir, instanceId, worktreeDir) {
+    const setupPath = join(packDir, "fixtures", instanceId, "setup.sh");
+    if (!existsSync(setupPath))
+        return { ok: true }; // No setup needed.
+    const r = spawnSync(setupPath, [], {
+        cwd: worktreeDir,
+        shell: true, // works for both .sh on POSIX and bash-as-shell on Windows
+        encoding: "utf-8",
+    });
+    if (r.status !== 0) {
+        return { ok: false, error: (r.stderr ?? "").slice(-500) };
+    }
+    return { ok: true };
+}
+function defaultOhEntry() {
+    return join(process.cwd(), "dist", "main.js");
+}
+function defaultRunArgs(opts) {
+    const args = [
+        opts.ohEntry,
+        "run",
+        "--bare",
+        "--output-format",
+        "stream-json",
+        "--no-session-persistence",
+        "--max-budget-usd",
+        String(opts.perTaskCostCap),
+        "--max-turns",
+        String(opts.maxTurns),
+        "--model",
+        opts.model,
+    ];
+    if (opts.fallbackModel)
+        args.push("--fallback-model", opts.fallbackModel);
+    args.push(opts.prompt);
+    return args;
+}
+function readHarnessVersion() {
+    try {
+        const pkgPath = join(process.cwd(), "package.json");
+        const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
+        return pkg.version ?? "0.0.0";
+    }
+    catch {
+        return "0.0.0";
+    }
+}
+function pathBaseName(p) {
+    const parts = p.split(/[\\/]/);
+    return parts[parts.length - 1] ?? "";
+}
+//# sourceMappingURL=orchestrator.js.map

package/dist/evals/pack-loader.d.ts

@@ -0,0 +1,29 @@
+/**
+ * oh evals — pack loader. Loads, validates, and resolves fixture paths
+ * for eval packs from disk.
+ *
+ * Bundled packs live under `data/evals/packs/<name>/`. User-installed packs
+ * live under `~/.oh/evals/packs/<name>/`. Bundled packs win precedence on
+ * name collision (so users can't shadow `swe-bench-lite-mini` accidentally).
+ */
+import type { EvalsPack, EvalsTask } from "./types.js";
+export declare function validatePack(packDir: string): {
+    ok: true;
+} | {
+    ok: false;
+    errors: string[];
+};
+export declare function loadPack(packDir: string): {
+    pack: EvalsPack;
+    tasks: EvalsTask[];
+};
+export declare function resolveFixturePath(packDir: string, instanceId: string): string;
+/** Returns names of packs found in bundled and user directories. Bundled wins on collision. */
+export declare function listAvailablePacks(): string[];
+/**
+ * Resolve a pack by name to its on-disk directory. Bundled packs (under
+ * the package's data/evals/packs/) win precedence over user packs
+ * (~/.oh/evals/packs/).
+ */
+export declare function resolvePackDir(packName: string): string | null;
+//# sourceMappingURL=pack-loader.d.ts.map

package/dist/evals/pack-loader.js

@@ -0,0 +1,153 @@
+/**
+ * oh evals — pack loader. Loads, validates, and resolves fixture paths
+ * for eval packs from disk.
+ *
+ * Bundled packs live under `data/evals/packs/<name>/`. User-installed packs
+ * live under `~/.oh/evals/packs/<name>/`. Bundled packs win precedence on
+ * name collision (so users can't shadow `swe-bench-lite-mini` accidentally).
+ */
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+const REQUIRED_PACK_FIELDS = [
+    "name",
+    "version",
+    "description",
+    "language",
+    "runner_requirements",
+    "default_test_command",
+    "instance_count",
+];
+const REQUIRED_TASK_FIELDS = [
+    "instance_id",
+    "repo",
+    "base_commit",
+    "problem_statement",
+    "FAIL_TO_PASS",
+    "PASS_TO_PASS",
+];
+export function validatePack(packDir) {
+    const errors = [];
+    const packJsonPath = join(packDir, "pack.json");
+    if (!existsSync(packJsonPath)) {
+        errors.push(`missing pack.json at ${packJsonPath}`);
+        return { ok: false, errors };
+    }
+    let pack;
+    try {
+        pack = JSON.parse(readFileSync(packJsonPath, "utf-8"));
+    }
+    catch (err) {
+        errors.push(`failed to parse pack.json: ${err.message}`);
+        return { ok: false, errors };
+    }
+    if (typeof pack !== "object" || pack === null) {
+        errors.push(`pack.json is not an object`);
+        return { ok: false, errors };
+    }
+    for (const field of REQUIRED_PACK_FIELDS) {
+        if (!(field in pack)) {
+            errors.push(`pack.json missing required field: ${field}`);
+        }
+    }
+    const instancesPath = join(packDir, "instances.jsonl");
+    if (!existsSync(instancesPath)) {
+        errors.push(`missing instances.jsonl at ${instancesPath}`);
+        return errors.length === 0 ? { ok: true } : { ok: false, errors };
+    }
+    const lines = readFileSync(instancesPath, "utf-8")
+        .split("\n")
+        .filter((l) => l.trim().length > 0);
+    for (let i = 0; i < lines.length; i++) {
+        const lineNo = i + 1;
+        let task;
+        try {
+            task = JSON.parse(lines[i]);
+        }
+        catch (err) {
+            errors.push(`instances.jsonl:${lineNo} parse error: ${err.message}`);
+            continue;
+        }
+        if (typeof task !== "object" || task === null) {
+            errors.push(`instances.jsonl:${lineNo} not an object`);
+            continue;
+        }
+        for (const field of REQUIRED_TASK_FIELDS) {
+            if (!(field in task)) {
+                errors.push(`instances.jsonl:${lineNo} missing required field: ${field}`);
+            }
+        }
+        const instanceId = task.instance_id;
+        if (typeof instanceId === "string") {
+            const fixtureDir = join(packDir, "fixtures", instanceId);
+            if (!existsSync(fixtureDir)) {
+                errors.push(`fixture dir missing for ${instanceId} at ${fixtureDir}`);
+            }
+            else {
+                if (!existsSync(join(fixtureDir, "repo.tar.zst"))) {
+                    errors.push(`fixture missing repo.tar.zst for ${instanceId}`);
+                }
+                if (!existsSync(join(fixtureDir, "setup.sh"))) {
+                    errors.push(`fixture missing setup.sh for ${instanceId}`);
+                }
+            }
+        }
+    }
+    return errors.length === 0 ? { ok: true } : { ok: false, errors };
+}
+export function loadPack(packDir) {
+    const r = validatePack(packDir);
+    if (!r.ok) {
+        throw new Error(`pack at ${packDir} failed validation:\n  - ${r.errors.join("\n  - ")}`);
+    }
+    const pack = JSON.parse(readFileSync(join(packDir, "pack.json"), "utf-8"));
+    const lines = readFileSync(join(packDir, "instances.jsonl"), "utf-8")
+        .split("\n")
+        .filter((l) => l.trim().length > 0);
+    const tasks = lines.map((l) => JSON.parse(l));
+    return { pack, tasks };
+}
+export function resolveFixturePath(packDir, instanceId) {
+    return join(packDir, "fixtures", instanceId);
+}
+/** Returns names of packs found in bundled and user directories. Bundled wins on collision. */
+export function listAvailablePacks() {
+    const seen = new Set();
+    const out = [];
+    for (const root of packSearchRoots()) {
+        if (!existsSync(root))
+            continue;
+        for (const entry of readdirSync(root)) {
+            const full = join(root, entry);
+            if (statSync(full).isDirectory() && existsSync(join(full, "pack.json")) && !seen.has(entry)) {
+                seen.add(entry);
+                out.push(entry);
+            }
+        }
+    }
+    return out;
+}
+/**
+ * Resolve a pack by name to its on-disk directory. Bundled packs (under
+ * the package's data/evals/packs/) win precedence over user packs
+ * (~/.oh/evals/packs/).
+ */
+export function resolvePackDir(packName) {
+    for (const root of packSearchRoots()) {
+        const candidate = join(root, packName);
+        if (existsSync(join(candidate, "pack.json")))
+            return candidate;
+    }
+    return null;
+}
+function packSearchRoots() {
+    // Bundled: packaged data/evals/packs/, located relative to this module.
+    // In dev: src/evals/pack-loader.ts → ../../data/evals/packs/
+    // In published build: dist/evals/pack-loader.js → ../../data/evals/packs/
+    const here = dirname(fileURLToPath(import.meta.url));
+    const bundled = join(here, "..", "..", "data", "evals", "packs");
+    const user = join(homedir(), ".oh", "evals", "packs");
+    return [bundled, user];
+}
+//# sourceMappingURL=pack-loader.js.map

package/dist/evals/run-writer.d.ts

@@ -0,0 +1,35 @@
+/**
+ * oh evals — run writer.
+ *
+ * Streams per-task results to disk atomically:
+ *  - results.jsonl   : append-only, one EvalsResult per line
+ *  - predictions.json: array, rewritten on each append, SWE-bench-submittable
+ *  - results.json    : merged + aggregates, written ONLY by finalize()
+ *
+ * Crash-safety: results.jsonl + predictions.json are valid up to the last
+ * successful append. `oh evals run --resume <run_id>` reads results.jsonl
+ * to determine completed instance_ids.
+ */
+import type { EvalsResult, RunArtifacts } from "./types.js";
+export type RunHeader = {
+    run_id: string;
+    pack: string;
+    pack_version: string;
+    model: string;
+    harness_version: string;
+    max_cost_usd: number;
+    started_at: string;
+};
+export declare class RunWriter {
+    private readonly runDir;
+    private readonly header;
+    private readonly results;
+    constructor(runDir: string, header: RunHeader);
+    appendResult(result: EvalsResult): void;
+    loadExistingResults(): EvalsResult[];
+    finalize(opts: {
+        partial: boolean;
+        finished_at: string;
+    }): RunArtifacts;
+}
+//# sourceMappingURL=run-writer.d.ts.map