@hona/openeval 0.5.5 → 0.5.6

package/VERIFICATION.md

@@ -92,6 +92,13 @@ evidence. Do not put evaluator scripts in candidate workspaces.
 
 ## Readable judgments and controls
 
+Retained code judgments stay scored when a metadata-only identity change leaves
+the exact self-contained executable unchanged. The reader verifies both recorded
+identities and rejects changed source, external package imports, unknown metadata,
+or a different execution protocol. This check makes no model calls and rewrites
+no evidence. A completed code verification is still required. Package/lockfile
+metadata alone must not make an unchanged historical scorecard appear unscored.
+
 `openeval prepare --output <new-directory>` assembles real candidate inputs and
 runs declared preparation in the candidate image, then archives the prepared
 workspace. `--only-eval` scopes it. This makes zero model calls, creates no

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.5.5",
+  "version": "0.5.6",
   "description": "Code and LLM evaluations with isolated agents, recorded OpenCode data, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/scores.ts

@@ -1,7 +1,9 @@
 import type { BenchmarkDefinition, JudgeRun, Slot } from "../types";
+import type { CodeJudgeDefinition } from "../judge-context";
 import { modelScore } from "../view";
 import { isScored } from "../judgment";
 import { categoryKey, inCategories } from "../criterion-categories";
+import { recordedCodeMatches } from "../infra/judging/code-source";
 
 /** Scores are derived from a single selection snapshot; every eval has equal weight. */
 export function benchmarkScores(
@@ -14,12 +16,23 @@ export function benchmarkScores(
   const evals = definition.evals.filter(
     (item) => !evalId || item.id === evalId,
   );
-  // A code judgment counts only when it ran the eval's current judge.ts to completion.
+  const matches = new WeakMap<CodeJudgeDefinition, WeakMap<CodeJudgeDefinition, boolean>>();
+  const equivalent = (expected: CodeJudgeDefinition, recorded: CodeJudgeDefinition | undefined) => {
+    if (!recorded) return false;
+    let previous = matches.get(expected);
+    if (!previous) { previous = new WeakMap(); matches.set(expected, previous); }
+    const cached = previous.get(recorded);
+    if (cached !== undefined) return cached;
+    const value = recordedCodeMatches(expected, recorded);
+    previous.set(recorded, value);
+    return value;
+  };
+  // A code judgment counts only after completed verification of the matching executable.
   const current = (item: (typeof evals)[number], slot: Slot) => {
     const judge = slot.judgeRunId ? index.get(slot.judgeRunId) : undefined;
     return !item.code ||
       (judge?.code?.state === "completed" &&
-        judge.input.code?.hash === item.code.hash)
+         equivalent(item.code, judge.input.code))
       ? judge
       : undefined;
   };

package/src/infra/judging/code-source.ts

@@ -10,6 +10,37 @@ const printer = new Bun.Transpiler({
   deadCodeElimination: false,
 });
 
+function portableDirectories(source: string) {
+  return source.replace(
+    /^\s*var __(?:dirname|filename) = .*$/gm,
+    (line) => line.replace(/"(?:[^"\\]|\\.)*"/g, '""'),
+  );
+}
+
+/** An unchanged self-contained executable can retain judgments whose older
+ * identity included host package manifests and lockfiles. This is a read-only
+ * identity check, not a rewrite of recorded code or a changed scoring rule.
+ * External packages still require their exact executable identity. */
+export function recordedCodeMatches(
+  expected: CodeJudgeDefinition,
+  recorded: CodeJudgeDefinition | undefined,
+) {
+  if (!recorded) return false;
+  if (recorded.hash === expected.hash) return true;
+  if (!expected.source.trim() || expected.source !== recorded.source ||
+      Object.keys(expected.dependencies).length !== 0) return false;
+  const imports = new Bun.Transpiler({ loader: "js" }).scan(expected.source).imports;
+  if (imports.some(item => !item.path.startsWith("node:") && !item.path.startsWith("bun:"))) return false;
+  const metadata = Object.keys(recorded.dependencies);
+  if (!metadata.length || metadata.some(path =>
+    !/(?:^|[\\/])(?:package\.json|bun\.lockb?|package-lock\.json|pnpm-lock\.yaml|yarn\.lock)$/.test(path))) return false;
+  // Verify both identities rather than accepting an arbitrary mismatched hash.
+  return CODE_JUDGE_PROTOCOL === 1 &&
+    recorded.hash === fingerprint({ source: recorded.source, dependencies: recorded.dependencies }) &&
+    expected.hash === fingerprint({ protocol: CODE_JUDGE_PROTOCOL,
+      source: printer.transformSync(portableDirectories(expected.source)) });
+}
+
 /** The package that provides an external import, as `name@version/path`. */
 async function packageSpecifier(path: string, from: string) {
   for (let directory = dirname(path); ; directory = dirname(directory)) {
@@ -93,10 +124,7 @@ export async function compileCodeJudge(
     );
   }
   // Bun writes __dirname and __filename as absolute paths. Like other runtime file inputs, they are not identity.
-  portable = portable.replace(
-    /^\s*var __(?:dirname|filename) = .*$/gm,
-    (line) => line.replace(/"(?:[^"\\]|\\.)*"/g, '""'),
-  );
+  portable = portableDirectories(portable);
   return {
     file,
     source,