@hona/openeval 0.5.2 → 0.5.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hona/openeval",
-  "version": "0.5.2",
+  "version": "0.5.3",
   "description": "Code and LLM evaluations with isolated agents, recorded OpenCode data, and a results viewer",
   "license": "MIT",
   "repository": {

package/src/app/read-results.ts

@@ -72,7 +72,22 @@ export const evalResultId = (benchmarkId: string, evalId: string) =>
 export class ResultReader {
   private paths = new Map<string, string>();
   private runtimes = new Map<string, RuntimeInterval[]>();
+  private finishedJudges = new Map<string, Map<string, JudgeRun>>();
   constructor(readonly root: string) {}
+  /** Every judge run; parses only running or unseen records, since finalized runs never change. */
+  private judgeRunsOf(results: Results) {
+    const cache =
+      this.finishedJudges.get(results.path) ?? new Map<string, JudgeRun>();
+    this.finishedJudges.set(results.path, cache);
+    return results.judgeRunStates().flatMap(({ id }) => {
+      const cached = cache.get(id);
+      if (cached) return [cached];
+      const run = results.judgeRun(id);
+      if (!run) return [];
+      if (run.state !== "running") cache.set(id, run);
+      return [run];
+    });
+  }
   private runtime(
     run: EvalRun | JudgeRun,
     results: Results,
@@ -93,7 +108,7 @@ export class ResultReader {
         scheduled(slot, benchmark, now);
       if (current && "evalRunId" in run.input)
         current =
-          (judges ?? results.judgeRuns())
+          (judges ?? this.judgeRunsOf(results))
             .filter(
               (judge) =>
                 judge.input.evalRunId === candidate!.id &&
@@ -272,25 +287,17 @@ export class ResultReader {
       (item) => !evalId || item.id === evalId,
     );
     if (!evals.length) throw new Error("Unknown eval");
-    const scores = results.readSnapshot(() =>
-      benchmarkScores(
-        run.definition,
-        results.slots(),
-        results.judgeRuns(),
-        evalId,
-      ),
+    const [slots, allJudges] = results.readSnapshot(
+      () => [results.slots(), this.judgeRunsOf(results)] as const,
     );
+    const scores = benchmarkScores(run.definition, slots, allJudges, evalId);
     // Runtime and cost describe the work behind the active selections, not retired history.
     const active = new Set(
-      results
-        .slots()
-        .filter((slot) => slot.active)
-        .map((slot) => slot.id),
+      slots.filter((slot) => slot.active).map((slot) => slot.id),
     );
     const candidates = results
-        .evalRuns()
-        .filter((candidate) => active.has(candidate.slotId)),
-      allJudges = results.judgeRuns();
+      .evalRuns()
+      .filter((candidate) => active.has(candidate.slotId));
     const now = Date.now(),
       candidateIndex = new Map(candidates.map((run) => [run.id, run]));
     const evalRuntime = new Map<string, RuntimeInterval[]>();
@@ -380,7 +387,7 @@ export class ResultReader {
   private liveRuns(results: Results, evalIds: readonly string[]) {
     const benchmark = results.benchmark!,
       executions = results.evalRuns(),
-      judges = results.judgeRuns(),
+      judges = this.judgeRunsOf(results),
       now = Date.now();
     const byId = new Map(executions.map((run) => [run.id, run]));
     const bySlot = Map.groupBy(executions, (run) => run.slotId);

package/src/app/serve-results.ts

@@ -16,6 +16,16 @@ export async function serveResults(options: {
   await mkdir(root, { recursive: true });
   const reader = new ResultReader(root),
     encoder = new TextEncoder();
+  // Open tabs refresh on every change. Share one recent read so polling cannot saturate the server.
+  const recent = new Map<string, { at: number; value: Promise<unknown> }>();
+  const shared = <T>(key: string, read: () => Promise<T>): Promise<T> => {
+    const hit = recent.get(key);
+    if (hit && Date.now() - hit.at < 2000) return hit.value as Promise<T>;
+    const value = read();
+    recent.set(key, { at: Date.now(), value });
+    value.catch(() => recent.delete(key));
+    return value;
+  };
   const clients = new Set<ReadableStreamDefaultController<Uint8Array>>(),
     streams = new Set<() => void>();
   let changed: ReturnType<typeof setTimeout> | undefined;
@@ -44,7 +54,7 @@ export async function serveResults(options: {
         if (url.pathname === "/api/results")
           return Response.json(await reader.index());
         if (url.pathname === "/api/activity")
-          return Response.json(await reader.activity());
+          return Response.json(await shared("activity", () => reader.activity()));
         if (url.pathname === "/api/judge-checks")
           return Response.json(
             await reader.judgeAudit(
@@ -93,12 +103,14 @@ export async function serveResults(options: {
             ),
           );
         }
-        if (url.pathname === "/api/result")
+        if (url.pathname === "/api/result") {
+          const id = url.searchParams.get("id") ?? "";
           return Response.json(
             url.searchParams.get("format") === "runs"
-              ? await reader.evalRuns(url.searchParams.get("id") ?? "")
-              : await reader.summary(url.searchParams.get("id") ?? ""),
+              ? await shared(`runs:${id}`, () => reader.evalRuns(id))
+              : await shared(`summary:${id}`, () => reader.summary(id)),
           );
+        }
         if (url.pathname === "/api/events") {
           let owner: ReadableStreamDefaultController<Uint8Array>;
           return new Response(

package/src/infra/sqlite/index.ts

@@ -166,6 +166,13 @@ export class Results {
       .all()
       .map((row) => row.value);
   }
+  /** IDs and states in judgeRuns() order. Triggers keep finalized runs immutable, so readers can cache them. */
+  judgeRunStates() {
+    return this.sqlite.query("SELECT id, state FROM judge_runs").all() as Array<{
+      id: string;
+      state: JudgeRun["state"];
+    }>;
+  }
   evalRun(id: string) {
     return this.db
       .select()