@hona/openeval 0.5.6 → 0.5.8

package/VERIFICATION.md

@@ -93,7 +93,9 @@ evidence. Do not put evaluator scripts in candidate workspaces.
 ## Readable judgments and controls
 
 Retained code judgments stay scored when a metadata-only identity change leaves
-the exact self-contained executable unchanged. The reader verifies both recorded
+the comment-free self-contained executable unchanged. Generated bundle comments
+and debug IDs are excluded; string literals remain executable input.
+The reader verifies both recorded
 identities and rejects changed source, external package imports, unknown metadata,
 or a different execution protocol. This check makes no model calls and rewrites
 no evidence. A completed code verification is still required. Package/lockfile

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.5.6",
+  "version": "0.5.8",
   "description": "Code and LLM evaluations with isolated agents, recorded OpenCode data, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/model-name-metadata.ts

@@ -0,0 +1,10 @@
+import type { ModelNames, BenchmarkRun } from "../types";
+
+/** Reporting metadata only: no new attempts, selections, fingerprints, or heartbeat. */
+export function withModelNames(benchmark: BenchmarkRun, names: ModelNames): BenchmarkRun {
+  const known = new Set(benchmark.definition.models.map(model => model.split("#")[0]));
+  for (const [model, name] of Object.entries(names))
+    if (!known.has(model) || typeof name !== "string" || !name.trim())
+      throw new Error("Model display names must name a recorded model and have non-empty text");
+  return { ...benchmark, modelNames: { ...benchmark.modelNames, ...names } };
+}

package/src/app/refresh-model-names.ts

@@ -0,0 +1,21 @@
+import { resolve } from "node:path";
+import type { ModelNames } from "../types";
+import { Results } from "../infra/sqlite";
+import { readModelNames } from "../infra/containers/catalog";
+import { withModelNames } from "./model-name-metadata";
+
+/** Refresh catalog names, or apply names read from an authoritative provider catalog. */
+export async function refreshModelNames(directory: string, names?: ModelNames) {
+  using results = new Results(resolve(directory, "runner.db"));
+  const current = results.benchmark;
+  if (!current) throw new Error("No benchmark run in this directory");
+  if (current.mergedInto) throw new Error("Refresh names on the aggregate benchmark run");
+  const fresh = names ?? await readModelNames(current.definition, current.runtime.imageId);
+  // Re-read under the write lock: a live runner may have updated progress while
+  // catalog discovery was in flight. Never replace that progress or its heartbeat.
+  return results.transaction(() => {
+    const benchmark = withModelNames(results.benchmark!, fresh);
+    results.saveBenchmark(benchmark);
+    return { directory: resolve(directory), modelNames: benchmark.modelNames };
+  });
+}

package/src/cli.ts

@@ -15,6 +15,7 @@ import {
   buildVerificationImage,
   VERIFICATION_IMAGE,
   prepareInputs,
+  refreshModelNames,
 } from "./index";
 import type { ModelRef } from "./index";
 
@@ -45,6 +46,7 @@ Commands:
   merge-runs <to> <from> Merge results into one aggregate
   add-models <run>       Add models with repeated --model flags
   remove-models <run>    Remove active models while retaining their evidence
+  refresh-model-names <run> Refresh reporting names without running evals
   retry <run> <eval-run> Retry a candidate execution
   rejudge <run> <eval-run> Judge the saved evidence again
 
@@ -165,6 +167,9 @@ Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
       retiredSlots: result.retiredSlots,
       status: result.benchmark.state,
     }, null, 2));
+  } else if (command === "refresh-model-names") {
+    if (!args[1]) throw new Error("refresh-model-names requires a benchmark run directory");
+    console.log(JSON.stringify(await refreshModelNames(resolve(args[1])), null, 2));
   } else if (command === "retry" || command === "rejudge") {
     if (!args[1] || !args[2])
       throw new Error(
@@ -177,7 +182,7 @@ Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
     console.log(JSON.stringify(result, null, 2));
   } else
     throw new Error(
-      "Commands: image, plan, run, view, add-models, remove-models, merge-runs, retry, rejudge, snapshot",
+      "Commands: image, plan, run, view, add-models, remove-models, refresh-model-names, merge-runs, retry, rejudge, snapshot",
     );
 } catch (error) {
   console.error(error instanceof Error ? error.message : String(error));

package/src/index.ts

@@ -64,6 +64,7 @@ export {
 } from "./app/run-benchmark";
 export { addModels } from "./app/add-models";
 export { removeModels } from "./app/remove-models";
+export { refreshModelNames } from "./app/refresh-model-names";
 export type { CostEstimate } from "./app/cost-plan";
 export { mergeBenchmarkRuns } from "./app/merge-runs";
 export { retryEvalRun } from "./app/retry-run";

package/src/infra/judging/code-source.ts

@@ -18,7 +18,8 @@ function portableDirectories(source: string) {
 }
 
 /** An unchanged self-contained executable can retain judgments whose older
- * identity included host package manifests and lockfiles. This is a read-only
+ * identity included host package manifests and lockfiles. Generated comments
+ * and debug IDs are not executable input. This is a read-only
  * identity check, not a rewrite of recorded code or a changed scoring rule.
  * External packages still require their exact executable identity. */
 export function recordedCodeMatches(
@@ -27,7 +28,8 @@ export function recordedCodeMatches(
 ) {
   if (!recorded) return false;
   if (recorded.hash === expected.hash) return true;
-  if (!expected.source.trim() || expected.source !== recorded.source ||
+  if (!expected.source.trim() ||
+      printer.transformSync(expected.source) !== printer.transformSync(recorded.source) ||
       Object.keys(expected.dependencies).length !== 0) return false;
   const imports = new Bun.Transpiler({ loader: "js" }).scan(expected.source).imports;
   if (imports.some(item => !item.path.startsWith("node:") && !item.path.startsWith("bun:"))) return false;