@hona/openeval 0.5.5 → 0.5.7

package/VERIFICATION.md

@@ -92,6 +92,13 @@ evidence. Do not put evaluator scripts in candidate workspaces.
 
 ## Readable judgments and controls
 
+Retained code judgments stay scored when a metadata-only identity change leaves
+the exact self-contained executable unchanged. The reader verifies both recorded
+identities and rejects changed source, external package imports, unknown metadata,
+or a different execution protocol. This check makes no model calls and rewrites
+no evidence. A completed code verification is still required. Package/lockfile
+metadata alone must not make an unchanged historical scorecard appear unscored.
+
 `openeval prepare --output <new-directory>` assembles real candidate inputs and
 runs declared preparation in the candidate image, then archives the prepared
 workspace. `--only-eval` scopes it. This makes zero model calls, creates no

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.5.5",
+  "version": "0.5.7",
   "description": "Code and LLM evaluations with isolated agents, recorded OpenCode data, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/model-name-metadata.ts

@@ -0,0 +1,10 @@
+import type { ModelNames, BenchmarkRun } from "../types";
+
+/** Reporting metadata only: no new attempts, selections, fingerprints, or heartbeat. */
+export function withModelNames(benchmark: BenchmarkRun, names: ModelNames): BenchmarkRun {
+  const known = new Set(benchmark.definition.models.map(model => model.split("#")[0]));
+  for (const [model, name] of Object.entries(names))
+    if (!known.has(model) || typeof name !== "string" || !name.trim())
+      throw new Error("Model display names must name a recorded model and have non-empty text");
+  return { ...benchmark, modelNames: { ...benchmark.modelNames, ...names } };
+}

package/src/app/refresh-model-names.ts

@@ -0,0 +1,21 @@
+import { resolve } from "node:path";
+import type { ModelNames } from "../types";
+import { Results } from "../infra/sqlite";
+import { readModelNames } from "../infra/containers/catalog";
+import { withModelNames } from "./model-name-metadata";
+
+/** Refresh catalog names, or apply names read from an authoritative provider catalog. */
+export async function refreshModelNames(directory: string, names?: ModelNames) {
+  using results = new Results(resolve(directory, "runner.db"));
+  const current = results.benchmark;
+  if (!current) throw new Error("No benchmark run in this directory");
+  if (current.mergedInto) throw new Error("Refresh names on the aggregate benchmark run");
+  const fresh = names ?? await readModelNames(current.definition, current.runtime.imageId);
+  // Re-read under the write lock: a live runner may have updated progress while
+  // catalog discovery was in flight. Never replace that progress or its heartbeat.
+  return results.transaction(() => {
+    const benchmark = withModelNames(results.benchmark!, fresh);
+    results.saveBenchmark(benchmark);
+    return { directory: resolve(directory), modelNames: benchmark.modelNames };
+  });
+}

package/src/app/scores.ts

@@ -1,7 +1,9 @@
 import type { BenchmarkDefinition, JudgeRun, Slot } from "../types";
+import type { CodeJudgeDefinition } from "../judge-context";
 import { modelScore } from "../view";
 import { isScored } from "../judgment";
 import { categoryKey, inCategories } from "../criterion-categories";
+import { recordedCodeMatches } from "../infra/judging/code-source";
 
 /** Scores are derived from a single selection snapshot; every eval has equal weight. */
 export function benchmarkScores(
@@ -14,12 +16,23 @@ export function benchmarkScores(
   const evals = definition.evals.filter(
     (item) => !evalId || item.id === evalId,
   );
-  // A code judgment counts only when it ran the eval's current judge.ts to completion.
+  const matches = new WeakMap<CodeJudgeDefinition, WeakMap<CodeJudgeDefinition, boolean>>();
+  const equivalent = (expected: CodeJudgeDefinition, recorded: CodeJudgeDefinition | undefined) => {
+    if (!recorded) return false;
+    let previous = matches.get(expected);
+    if (!previous) { previous = new WeakMap(); matches.set(expected, previous); }
+    const cached = previous.get(recorded);
+    if (cached !== undefined) return cached;
+    const value = recordedCodeMatches(expected, recorded);
+    previous.set(recorded, value);
+    return value;
+  };
+  // A code judgment counts only after completed verification of the matching executable.
   const current = (item: (typeof evals)[number], slot: Slot) => {
     const judge = slot.judgeRunId ? index.get(slot.judgeRunId) : undefined;
     return !item.code ||
       (judge?.code?.state === "completed" &&
-        judge.input.code?.hash === item.code.hash)
+         equivalent(item.code, judge.input.code))
       ? judge
       : undefined;
   };

package/src/cli.ts

@@ -15,6 +15,7 @@ import {
   buildVerificationImage,
   VERIFICATION_IMAGE,
   prepareInputs,
+  refreshModelNames,
 } from "./index";
 import type { ModelRef } from "./index";
 
@@ -45,6 +46,7 @@ Commands:
   merge-runs <to> <from> Merge results into one aggregate
   add-models <run>       Add models with repeated --model flags
   remove-models <run>    Remove active models while retaining their evidence
+  refresh-model-names <run> Refresh reporting names without running evals
   retry <run> <eval-run> Retry a candidate execution
   rejudge <run> <eval-run> Judge the saved evidence again
 
@@ -165,6 +167,9 @@ Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
       retiredSlots: result.retiredSlots,
       status: result.benchmark.state,
     }, null, 2));
+  } else if (command === "refresh-model-names") {
+    if (!args[1]) throw new Error("refresh-model-names requires a benchmark run directory");
+    console.log(JSON.stringify(await refreshModelNames(resolve(args[1])), null, 2));
   } else if (command === "retry" || command === "rejudge") {
     if (!args[1] || !args[2])
       throw new Error(
@@ -177,7 +182,7 @@ Requires Bun 1.4.2+, Docker, and an authenticated OpenCode installation.`);
     console.log(JSON.stringify(result, null, 2));
   } else
     throw new Error(
-      "Commands: image, plan, run, view, add-models, remove-models, merge-runs, retry, rejudge, snapshot",
+      "Commands: image, plan, run, view, add-models, remove-models, refresh-model-names, merge-runs, retry, rejudge, snapshot",
     );
 } catch (error) {
   console.error(error instanceof Error ? error.message : String(error));

package/src/index.ts

@@ -64,6 +64,7 @@ export {
 } from "./app/run-benchmark";
 export { addModels } from "./app/add-models";
 export { removeModels } from "./app/remove-models";
+export { refreshModelNames } from "./app/refresh-model-names";
 export type { CostEstimate } from "./app/cost-plan";
 export { mergeBenchmarkRuns } from "./app/merge-runs";
 export { retryEvalRun } from "./app/retry-run";

package/src/infra/judging/code-source.ts

@@ -10,6 +10,37 @@ const printer = new Bun.Transpiler({
   deadCodeElimination: false,
 });
 
+function portableDirectories(source: string) {
+  return source.replace(
+    /^\s*var __(?:dirname|filename) = .*$/gm,
+    (line) => line.replace(/"(?:[^"\\]|\\.)*"/g, '""'),
+  );
+}
+
+/** An unchanged self-contained executable can retain judgments whose older
+ * identity included host package manifests and lockfiles. This is a read-only
+ * identity check, not a rewrite of recorded code or a changed scoring rule.
+ * External packages still require their exact executable identity. */
+export function recordedCodeMatches(
+  expected: CodeJudgeDefinition,
+  recorded: CodeJudgeDefinition | undefined,
+) {
+  if (!recorded) return false;
+  if (recorded.hash === expected.hash) return true;
+  if (!expected.source.trim() || expected.source !== recorded.source ||
+      Object.keys(expected.dependencies).length !== 0) return false;
+  const imports = new Bun.Transpiler({ loader: "js" }).scan(expected.source).imports;
+  if (imports.some(item => !item.path.startsWith("node:") && !item.path.startsWith("bun:"))) return false;
+  const metadata = Object.keys(recorded.dependencies);
+  if (!metadata.length || metadata.some(path =>
+    !/(?:^|[\\/])(?:package\.json|bun\.lockb?|package-lock\.json|pnpm-lock\.yaml|yarn\.lock)$/.test(path))) return false;
+  // Verify both identities rather than accepting an arbitrary mismatched hash.
+  return CODE_JUDGE_PROTOCOL === 1 &&
+    recorded.hash === fingerprint({ source: recorded.source, dependencies: recorded.dependencies }) &&
+    expected.hash === fingerprint({ protocol: CODE_JUDGE_PROTOCOL,
+      source: printer.transformSync(portableDirectories(expected.source)) });
+}
+
 /** The package that provides an external import, as `name@version/path`. */
 async function packageSpecifier(path: string, from: string) {
   for (let directory = dirname(path); ; directory = dirname(directory)) {
@@ -93,10 +124,7 @@ export async function compileCodeJudge(
     );
   }
   // Bun writes __dirname and __filename as absolute paths. Like other runtime file inputs, they are not identity.
-  portable = portable.replace(
-    /^\s*var __(?:dirname|filename) = .*$/gm,
-    (line) => line.replace(/"(?:[^"\\]|\\.)*"/g, '""'),
-  );
+  portable = portableDirectories(portable);
   return {
     file,
     source,