@hona/openeval 0.2.1 → 0.2.2

package/README.md

@@ -90,6 +90,18 @@ bunx --bun @hona/openeval view
 The viewer opens at **http://127.0.0.1:4173**. `run` resumes the same aggregate;
 scope flags select work while retaining existing scores.
 
+To remove a model from an existing aggregate, remove its entry from
+`benchmark.ts`, then retire its active selections:
+
+```sh
+bunx --bun @hona/openeval snapshot ./results/RUN before-model-removal
+bunx --bun @hona/openeval remove-models ./results/RUN --model provider/retired-model
+```
+
+This retains the model's recorded executions, judgments, and artifacts. It
+does not run candidates or judges, and requires a stopped benchmark run.
+Models still declared in `benchmark.ts` can be added back by a later `run`.
+
 ```mermaid
 flowchart LR
   P["prompt.md"] --> C["Isolated candidate"] --> E["Recording"]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "Typed prompt-and-judge evaluations with isolated agents, recorded evidence, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/remove-models.ts

@@ -0,0 +1,60 @@
+import { resolve } from "node:path";
+import type { ModelRef } from "../types";
+import { Results } from "../infra/sqlite";
+import { modelRef } from "./load-benchmark";
+import { readBenchmarkRun } from "./read-run";
+import { finishBenchmark } from "./run-benchmark";
+
+/** Retire active model selections while preserving every execution and artifact. */
+export async function removeModels(
+  directory: string,
+  models: readonly ModelRef[],
+) {
+  const root = resolve(directory);
+  const removed = [...new Set(models.map(modelRef))];
+  if (!removed.length) throw new Error("Select models to remove");
+  const before = readBenchmarkRun(root).benchmark;
+
+  using results = new Results(resolve(root, "runner.db"));
+  return results.transaction(() => {
+    const current = results.benchmark!;
+    if (current.id !== before.id || current.mergedInto)
+      throw new Error(
+        "Benchmark was replaced or merged; use its current aggregate",
+      );
+    if (current.state === "running")
+      throw new Error(
+        "Cannot remove models while the benchmark run is running",
+      );
+    if (removed.some((model) => !current.definition.models.includes(model)))
+      throw new Error("Select models present in this benchmark run");
+    const remaining = current.definition.models.filter(
+      (model) => !removed.includes(model),
+    );
+    if (!remaining.length)
+      throw new Error("Keep at least one model in the benchmark run");
+    const retired = results
+      .slots()
+      .filter((slot) => slot.active && removed.includes(slot.model));
+    for (const slot of retired) results.select({ ...slot, active: false });
+    const definition = { ...current.definition, models: remaining };
+    results.saveBenchmark({
+      ...current,
+      definition,
+      execution: { ...current.execution, deferred: 0 },
+    });
+    const benchmark = finishBenchmark({
+      directory: root,
+      definition,
+      runtime: current.runtime,
+      results,
+    });
+    results.saveBenchmark(benchmark, true);
+    return {
+      directory: root,
+      removed,
+      retiredSlots: retired.length,
+      benchmark,
+    };
+  });
+}

package/src/cli.ts

@@ -6,6 +6,7 @@ import {
   loadBenchmark,
   buildImage,
   addModels,
+  removeModels,
   retryEvalRun,
   judgeRun,
   serveResults,
@@ -39,6 +40,7 @@ Commands:
   snapshot <run> <name>  Export a score snapshot
   merge-runs <to> <from> Merge results into one aggregate
   add-models <run>       Add models with repeated --model flags
+  remove-models <run>    Remove active models while retaining their evidence
   retry <run> <eval-run> Retry a candidate execution
   rejudge <run> <eval-run> Judge the saved evidence again
 
@@ -46,7 +48,7 @@ Options:
   --benchmark <dir>      Benchmark directory (default: current directory)
   --run <dir>            Resume a specific result directory
   --new                 Start a new result
-  --model <provider/id>  Add a model to the benchmark
+  --model <provider/id>  Model to add or remove (repeatable)
   --only-model <ref>     Execute only this model (repeatable)
   --only-eval <id>       Execute only this eval (repeatable)
   --only-repetition <n>  Execute only this repetition (repeatable)
@@ -125,6 +127,16 @@ Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
     if (!args[1])
       throw new Error("add-models requires a benchmark run directory");
     console.log((await addModels(resolve(args[1]), models())).directory);
+  } else if (command === "remove-models") {
+    if (!args[1])
+      throw new Error("remove-models requires a benchmark run directory");
+    const result = await removeModels(resolve(args[1]), models());
+    console.log(JSON.stringify({
+      directory: result.directory,
+      removed: result.removed,
+      retiredSlots: result.retiredSlots,
+      status: result.benchmark.state,
+    }, null, 2));
   } else if (command === "retry" || command === "rejudge") {
     if (!args[1] || !args[2])
       throw new Error(
@@ -137,7 +149,7 @@ Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
     console.log(JSON.stringify(result, null, 2));
   } else
     throw new Error(
-      "Commands: image, plan, run, view, add-models, merge-runs, retry, rejudge, snapshot",
+      "Commands: image, plan, run, view, add-models, remove-models, merge-runs, retry, rejudge, snapshot",
     );
 } catch (error) {
   console.error(error instanceof Error ? error.message : String(error));

package/src/index.ts

@@ -38,6 +38,7 @@ export {
   type RunBenchmarkOptions,
 } from "./app/run-benchmark";
 export { addModels } from "./app/add-models";
+export { removeModels } from "./app/remove-models";
 export type { CostEstimate } from "./app/cost-plan";
 export { mergeBenchmarkRuns } from "./app/merge-runs";
 export { retryEvalRun } from "./app/retry-run";