@hona/openeval 0.2.0

package/src/app/judge-evidence.ts

@@ -0,0 +1,71 @@
+import { resolve, dirname } from "node:path";
+import type { EvidenceRef, Judge, JudgeRunInput, ToolCall } from "../types";
+import { executeJudge, judgingFingerprint } from "../infra/judging";
+import { EvidenceCapture } from "../infra/evidence";
+import { JUDGE_PROTOCOL, rubricMetrics } from "../judgment";
+import { JUDGE_AGENT } from "../infra/judging/agent";
+import { writeJson } from "../infra/files";
+import { savedJudgeFingerprint } from "./input-fingerprints";
+import { mkdir, mkdtemp, rm } from "node:fs/promises";
+
+/** Inspect retained evidence without changing benchmark selections.
+ * Useful for calibration and second opinions: https://arxiv.org/abs/2410.12784
+ */
+export async function judgeEvidence(options: {
+  evidence: EvidenceRef;
+  rubric: string;
+  judge: Judge;
+  directory: string;
+}) {
+  const directory = resolve(options.directory);
+  if (await Bun.file(resolve(directory, "input.json")).exists())
+    throw new Error("Choose a new judge evidence directory");
+  const request: Omit<JudgeRunInput, "judgeHash"> = {
+    evalRunId: "retained-evidence",
+    evidence: {
+      ...options.evidence,
+      directory: resolve(options.evidence.directory),
+    },
+    rubric: options.rubric,
+    agent: JUDGE_AGENT,
+    model: options.judge.model,
+    timeoutMs: options.judge.timeoutMs ?? 600_000,
+    websearch: options.judge.websearch ?? false,
+    metrics: rubricMetrics(options.rubric),
+    protocol: JUDGE_PROTOCOL,
+    mode: "final" as const,
+    runtimeHash: await judgingFingerprint(),
+  };
+  const input: JudgeRunInput = {
+    ...request,
+    judgeHash: savedJudgeFingerprint(request),
+  };
+  await writeJson(resolve(directory, "input.json"), input);
+  const result = await executeJudge(input, directory, () => {});
+  await writeJson(resolve(directory, "result.json"), result);
+  return result;
+}
+
+/** Record an explicitly supplied calibration example. No model or tool is executed. */
+export async function recordEvidence(options: {
+  directory: string;
+  prompt: string;
+  response: string;
+  tools?: ToolCall[];
+}): Promise<EvidenceRef> {
+  const directory = resolve(options.directory);
+  await mkdir(dirname(directory), { recursive: true });
+  const workspace = await mkdtemp(resolve(dirname(directory), ".recording-"));
+  try {
+    const capture = await EvidenceCapture.create(directory);
+    const result = await capture.finish({
+      prompt: options.prompt,
+      response: { text: options.response },
+      tools: options.tools ?? [],
+      workspace,
+    });
+    return { directory, hash: result.sha256 };
+  } finally {
+    await rm(workspace, { recursive: true, force: true });
+  }
+}

package/src/app/judge-run.ts

@@ -0,0 +1,89 @@
+import { resolve, relative } from "node:path";
+import type { EvalRun, JudgeRun } from "../types";
+import type { ExecutionContext } from "./context";
+import { executeJudge } from "../infra/judging";
+import { errorMessage } from "../infra/files";
+import { canJudgeEval } from "./eval-state";
+import { JUDGE_PROTOCOL } from "../judgment";
+import { JUDGE_AGENT } from "../infra/judging/agent";
+
+/** A new judge session references existing evidence and becomes active only after success. */
+export async function judgeEvalRun(
+  context: ExecutionContext,
+  candidate: EvalRun,
+  monitoring: Pick<JudgeRun, "monitorError" | "monitorErrorAt"> = {},
+): Promise<JudgeRun> {
+  if (!canJudgeEval(candidate))
+    throw new Error(
+      "Judging requires finalized evidence from a completed, stopped, or timed-out eval run",
+    );
+  const slot = context.results.slot(candidate.slotId)!;
+  const definition = context.definition.evals.find(
+    (item) => item.id === slot.evalId,
+  )!;
+  const run = context.results.startJudge({
+    evalRunId: candidate.id,
+    evidence: candidate.evidence,
+    rubric: definition.judge,
+    agent: JUDGE_AGENT,
+    model: context.definition.judge.model,
+    judgeHash: slot.judgeHash,
+    timeoutMs: context.definition.judge.timeoutMs,
+    websearch: context.definition.judge.websearch,
+    mode: "final",
+    runtimeHash: context.runtime.judgeHash,
+    protocol: JUDGE_PROTOCOL,
+    metrics: definition.metrics,
+  });
+  const directory = resolve(context.directory, "judge-runs", run.id);
+  let completed: JudgeRun;
+  try {
+    const { result, session, judgment } = await executeJudge(
+      {
+        ...run.input,
+        evidence: {
+          ...run.input.evidence,
+          directory: resolve(context.directory, run.input.evidence.directory),
+        },
+      },
+      directory,
+      (event) => {
+        const record = {
+          executionId: run.id,
+          stage: "judge" as const,
+          time: new Date().toISOString(),
+          event,
+        };
+        context.results.append(record);
+        context.onEvent?.(record);
+      },
+    );
+    completed = {
+      ...run,
+      ...monitoring,
+      state: result.state,
+      judgment,
+      session: {
+        ...session,
+        database: relative(
+          context.directory,
+          resolve(directory, session.database),
+        ),
+      },
+      error: result.error,
+      completedAt: new Date().toISOString(),
+      elapsedMs: Date.now() - Date.parse(run.startedAt),
+    };
+  } catch (error) {
+    completed = {
+      ...run,
+      ...monitoring,
+      state: "failed",
+      error: errorMessage(error),
+      completedAt: new Date().toISOString(),
+      elapsedMs: Date.now() - Date.parse(run.startedAt),
+    };
+  }
+  context.results.finishJudge(completed);
+  return completed;
+}

package/src/app/load-benchmark.ts

@@ -0,0 +1,267 @@
+import { readdir, stat } from "node:fs/promises";
+import { basename, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import type {
+  Benchmark,
+  BenchmarkDefinition,
+  Eval,
+  EvalDefinition,
+  ModelRef,
+} from "../types";
+import { CANDIDATE_TIMEOUT_MS } from "../types";
+import { rubricMetrics } from "../judgment";
+import { monitorPolicy } from "./monitor-policy";
+import {
+  fingerprint,
+  hash,
+  relativePath,
+  treeHash,
+  contained,
+} from "../infra/files";
+
+const positive = (value: unknown, fallback: number, label: string) => {
+  const result = value ?? fallback;
+  if (typeof result !== "number" || !Number.isInteger(result) || result < 1)
+    throw new Error(`${label} must be a positive integer`);
+  return result;
+};
+export const modelRef = (value: unknown): ModelRef => {
+  if (
+    typeof value !== "string" ||
+    !/^[\w.-]+\/[^\s/#]+(?:#[\w.-]+)?$/.test(value)
+  )
+    throw new Error(`Invalid model reference: ${String(value)}`);
+  return value as ModelRef;
+};
+async function declaration<T>(path: string): Promise<T> {
+  const url = pathToFileURL(path);
+  url.searchParams.set("version", hash(await Bun.file(path).bytes()));
+  return (await import(url.href)).default;
+}
+async function loadEval(directory: string): Promise<EvalDefinition> {
+  const id = basename(directory),
+    prompt = Bun.file(resolve(directory, "prompt.md")),
+    judge = Bun.file(resolve(directory, "judge.md"));
+  if (!(await prompt.exists()) || !(await judge.exists()))
+    throw new Error(`${id} requires prompt.md and judge.md`);
+  const settings = (await Bun.file(resolve(directory, "eval.ts")).exists())
+    ? await declaration<Eval>(resolve(directory, "eval.ts"))
+    : {};
+  if (!settings || typeof settings !== "object")
+    throw new Error(`${id}/eval.ts must export a declaration`);
+  if (
+    Object.keys(settings).some(
+      (key) => !["workspace", "prepare", "earlyStop"].includes(key),
+    )
+  )
+    throw new Error(`${id}/eval.ts has unsupported settings`);
+  monitorPolicy(settings.earlyStop);
+  const source: unknown[] = [];
+  if (settings.workspace) {
+    const workspace = settings.workspace;
+    if (
+      Object.keys(workspace).some(
+        (key) =>
+          ![
+            "repository",
+            "revisions",
+            "ref",
+            "commit",
+            "checkout",
+            "remote",
+            "overlay",
+          ].includes(key),
+      )
+    )
+      throw new Error(`${id}: workspace contains unsupported settings`);
+    if (!/^[\w./-]+$/.test(workspace.ref))
+      throw new Error(`${id}: workspace ref is invalid`);
+    if (workspace.commit && !/^[a-f0-9]{40}$/.test(workspace.commit))
+      throw new Error(`${id}: workspace commit must be a full SHA`);
+    if (
+      [workspace.repository, workspace.revisions].filter(Boolean).length !== 1
+    )
+      throw new Error(
+        `${id}: choose one workspace source: repository or revisions`,
+      );
+    if (workspace.repository) {
+      const url = new URL(workspace.repository);
+      if (
+        url.protocol !== "https:" ||
+        url.username ||
+        url.password ||
+        !workspace.commit
+      )
+        throw new Error(
+          `${id}: repository requires a credential-free HTTPS URL and a full commit SHA`,
+        );
+      source.push({
+        repository: workspace.repository,
+        commit: workspace.commit,
+      });
+    }
+    if (workspace.revisions) {
+      if (
+        !workspace.revisions.length ||
+        !workspace.revisions.some((revision) => revision.ref === workspace.ref)
+      )
+        throw new Error(`${id}: revisions must include the initial ref`);
+      for (const revision of workspace.revisions) {
+        if (
+          !/^[\w.-]+$/.test(revision.ref) ||
+          revision.ref === "preparation" ||
+          !revision.message
+        )
+          throw new Error(`${id}: invalid revision ref/message`);
+        source.push(
+          await treeHash(
+            contained(
+              directory,
+              relativePath(revision.directory, "Revision files"),
+            ),
+          ),
+        );
+      }
+    }
+    if (workspace.overlay)
+      source.push(
+        await treeHash(
+          contained(
+            directory,
+            relativePath(workspace.overlay, "Workspace overlay"),
+          ),
+        ),
+      );
+    for (const name of [
+      workspace.checkout ?? "checkout",
+      workspace.remote ?? "upstream",
+    ])
+      if (!/^[\w-]+$/.test(name))
+        throw new Error(`${id}: checkout and remote must be simple names`);
+  } else {
+    const workspace = resolve(directory, "workspace");
+    if (
+      await stat(workspace)
+        .then((s) => s.isDirectory())
+        .catch(() => false)
+    )
+      source.push(await treeHash(workspace));
+  }
+  for (const step of settings.prepare ?? []) {
+    relativePath(step.cwd, "Preparation working directory");
+    if (
+      !Array.isArray(step.argv) ||
+      !step.argv.length ||
+      step.argv.some((arg) => typeof arg !== "string" || !arg)
+    )
+      throw new Error(`${id}: preparation requires argv`);
+  }
+  const promptText = await prompt.text(),
+    judgeText = await judge.text();
+  if (!promptText.trim() || !judgeText.trim())
+    throw new Error(`${id}: prompt and judge must not be empty`);
+  return {
+    id,
+    directory,
+    prompt: promptText,
+    judge: judgeText,
+    settings,
+    sourceHash: fingerprint({
+      settings: { workspace: settings.workspace, prepare: settings.prepare },
+      source,
+    }),
+    judgeHash: hash(judgeText),
+    metrics: rubricMetrics(judgeText),
+    name: /^# (.+)$/m.exec(judgeText)?.[1] ?? id,
+  };
+}
+
+export async function loadBenchmark(
+  path: string,
+): Promise<BenchmarkDefinition> {
+  const directory = resolve(
+    path.endsWith("benchmark.ts") ? resolve(path, "..") : path,
+  );
+  const declarationFile = resolve(directory, "benchmark.ts");
+  const definition = await declaration<Benchmark>(declarationFile);
+  if (
+    !definition ||
+    !Array.isArray(definition.models) ||
+    !definition.models.length ||
+    !definition.judge
+  )
+    throw new Error("benchmark.ts requires models and a judge");
+  if (
+    Object.keys(definition).some(
+      (key) =>
+        ![
+          "name",
+          "models",
+          "judge",
+          "repetitions",
+          "concurrency",
+          "candidate",
+          "container",
+        ].includes(key),
+    )
+  )
+    throw new Error("benchmark.ts contains unsupported settings");
+  const models = definition.models.map(modelRef);
+  if (new Set(models).size !== models.length)
+    throw new Error("Benchmark contains duplicate models");
+  const timeoutMs = positive(
+    definition.candidate?.timeoutMs,
+    CANDIDATE_TIMEOUT_MS,
+    "Candidate timeout",
+  );
+  if (timeoutMs > CANDIDATE_TIMEOUT_MS)
+    throw new Error("Candidate timeout cannot exceed 45 minutes");
+  const evalRoot = resolve(directory, "evals");
+  const directories = (await readdir(evalRoot, { withFileTypes: true }))
+    .filter((entry) => entry.isDirectory() && !entry.name.startsWith("."))
+    .sort((a, b) => a.name.localeCompare(b.name));
+  if (!directories.length) throw new Error("Benchmark has no eval folders");
+  const evals = await Promise.all(
+    directories.map((entry) => loadEval(resolve(evalRoot, entry.name))),
+  );
+  const concurrency = positive(definition.concurrency, 10, "Concurrency");
+  if (concurrency < 2 && evals.some((item) => item.settings.earlyStop))
+    throw new Error(
+      "Early stopping requires concurrency of at least 2 for eval and judge work",
+    );
+  const engine = definition.container?.engine ?? "docker";
+  if (engine !== "docker" && engine !== "podman")
+    throw new Error("Container engine must be docker or podman");
+  for (const search of [
+    definition.candidate?.websearch,
+    definition.judge.websearch,
+  ])
+    if (search !== undefined && search !== false && search !== "exa")
+      throw new Error("Websearch must be exa or false");
+  return {
+    name: definition.name ?? basename(directory),
+    directory,
+    models,
+    evals,
+    repetitions: positive(definition.repetitions, 3, "Repetitions"),
+    concurrency,
+    candidate: {
+      timeoutMs,
+      websearch: definition.candidate?.websearch ?? "exa",
+      ...(definition.candidate?.providers
+        ? { providers: definition.candidate.providers }
+        : {}),
+    },
+    judge: {
+      model: modelRef(definition.judge.model),
+      timeoutMs: positive(definition.judge.timeoutMs, 600_000, "Judge timeout"),
+      websearch: definition.judge.websearch ?? "exa",
+    },
+    container: {
+      engine,
+      image: definition.container?.image ?? "openeval-runtime:0.2",
+      cpus: positive(definition.container?.cpus, 2, "CPU count"),
+      memoryMiB: positive(definition.container?.memoryMiB, 4096, "Memory"),
+    },
+  };
+}

package/src/app/merge-runs.ts

@@ -0,0 +1,117 @@
+import { cp, mkdir, readdir } from "node:fs/promises";
+import { resolve, relative } from "node:path";
+import { Results } from "../infra/sqlite";
+import { CandidateEvidence } from "../infra/evidence";
+import { hash } from "../infra/files";
+import { loadBenchmark } from "./load-benchmark";
+import { finishBenchmark } from "./run-benchmark";
+
+/** Consolidate same-benchmark evidence without repeating execution or rewriting finalized records. */
+export async function mergeBenchmarkRuns(
+  targetPath: string,
+  sourcePath: string,
+) {
+  const directory = resolve(targetPath),
+    sourceDirectory = resolve(sourcePath);
+  if (directory === sourceDirectory)
+    throw new Error("Select two different run directories");
+  let sourceId: string, sourceHash: string;
+  const backup = resolve(directory, "maintenance", `merge-${Date.now()}`);
+  {
+    using target = new Results(resolve(directory, "runner.db"), true);
+    using source = new Results(resolve(sourceDirectory, "runner.db"), true);
+    if (!target.benchmark || !source.benchmark)
+      throw new Error("Both directories must contain benchmark runs");
+    if (source.benchmark.mergedInto === target.benchmark.id)
+      return { directory, benchmark: target.benchmark };
+    if (source.benchmark.mergedInto || target.benchmark.mergedInto)
+      throw new Error("Select the current aggregate run");
+    if ([target.benchmark.state, source.benchmark.state].includes("running"))
+      throw new Error("Wait for active work before merging");
+    const key = (path: string) =>
+      process.platform === "win32"
+        ? resolve(path).toLowerCase()
+        : resolve(path);
+    if (key(target.benchmark.source) !== key(source.benchmark.source))
+      throw new Error("Runs belong to different benchmarks");
+    sourceId = source.benchmark.id;
+    await target.backup(resolve(backup, "target.db"));
+    await source.backup(resolve(backup, "source.db"));
+    sourceHash = hash(await Bun.file(resolve(backup, "source.db")).bytes());
+  }
+  for (const folder of ["eval-runs", "judge-runs", "inputs"]) {
+    const path = resolve(sourceDirectory, folder);
+    if (!(await readdir(path).catch(() => undefined))) continue;
+    await mkdir(resolve(directory, folder), { recursive: true });
+    await cp(path, resolve(directory, folder), {
+      recursive: true,
+      force: false,
+      errorOnExist: false,
+      verbatimSymlinks: true,
+    });
+  }
+  using target = new Results(resolve(directory, "runner.db"));
+  using source = new Results(resolve(sourceDirectory, "runner.db"));
+  for (const run of [...source.evalRuns(), ...source.judgeRuns()]) {
+    if (
+      run.session &&
+      hash(await Bun.file(resolve(directory, run.session.database)).bytes()) !==
+        run.session.databaseHash
+    )
+      throw new Error(`Copied session archive does not match ${run.id}`);
+    if ("slotId" in run && run.evidence)
+      await CandidateEvidence.open(
+        resolve(directory, run.evidence.directory),
+        run.evidence.hash,
+      );
+  }
+  for (const judge of source.judgeRuns())
+    for (const check of source.judgeChecks(judge.id))
+      await CandidateEvidence.open(
+        resolve(directory, check.checkpoint.directory),
+        check.checkpoint.hash,
+      );
+  const previous = target.benchmark!;
+  const current = await loadBenchmark(previous.source);
+  const definition = {
+    ...current,
+    models: [
+      ...new Set([
+        ...previous.definition.models,
+        ...source.benchmark!.definition.models,
+        ...current.models,
+      ]),
+    ],
+  };
+  target.mergeFinalized(source);
+  target.saveBenchmark(
+    {
+      ...previous,
+      definition,
+      state: "incomplete",
+      updatedAt: new Date().toISOString(),
+      sources: [
+        ...(previous.sources ?? []),
+        {
+          id: sourceId,
+          directory: relative(directory, sourceDirectory),
+          mergedAt: new Date().toISOString(),
+          databaseHash: sourceHash,
+        },
+      ],
+    },
+    true,
+  );
+  const benchmark = finishBenchmark({
+    directory,
+    definition,
+    runtime: previous.runtime,
+    results: target,
+  });
+  source.saveBenchmark({
+    ...source.benchmark!,
+    mergedInto: previous.id,
+    updatedAt: new Date().toISOString(),
+  });
+  return { directory, benchmark };
+}

package/src/app/monitor-policy.ts

@@ -0,0 +1,49 @@
+import type { Eval, ModelRef, MonitorPolicy } from "../types";
+
+/** Bound evaluator overhead; monitoring observes but never steers candidate work.
+ * https://deepswe.datacurve.ai/blog/deepswe#beyond-pass-rate
+ */
+export function monitorPolicy(
+  setting: Eval["earlyStop"],
+  model?: ModelRef,
+): MonitorPolicy | undefined {
+  if (setting === undefined || setting === false) return;
+  const options = setting === true ? {} : setting;
+  if (
+    typeof options !== "object" ||
+    Object.keys(options).some(
+      (key) =>
+        !["minIntervalMs", "maxChecks", "maxCostUSD", "onlyModels"].includes(
+          key,
+        ),
+    )
+  )
+    throw new Error("Invalid earlyStop policy");
+  const policy = {
+    minIntervalMs: options.minIntervalMs ?? 45_000,
+    maxChecks: options.maxChecks ?? 12,
+    maxCostUSD: options.maxCostUSD ?? 1,
+  };
+  if (
+    !Number.isSafeInteger(policy.minIntervalMs) ||
+    policy.minIntervalMs < 0 ||
+    !Number.isSafeInteger(policy.maxChecks) ||
+    policy.maxChecks < 1 ||
+    !Number.isFinite(policy.maxCostUSD) ||
+    policy.maxCostUSD < 0
+  )
+    throw new Error("Invalid earlyStop interval, check limit, or cost limit");
+  if (
+    options.onlyModels &&
+    (!Array.isArray(options.onlyModels) ||
+      !options.onlyModels.length ||
+      options.onlyModels.some(
+        (value) =>
+          typeof value !== "string" || !/^[\w.-]+\/[^\s/]+$/.test(value),
+      ))
+  )
+    throw new Error("earlyStop.onlyModels must list model references");
+  if (model && options.onlyModels && !options.onlyModels.includes(model))
+    return;
+  return policy;
+}