@hona/openeval 0.2.0

package/src/infra/judging/observer.ts

@@ -0,0 +1,168 @@
+import { OpenCode } from "@opencode/sdk";
+import { mkdir } from "node:fs/promises";
+import { resolve } from "node:path";
+import type { EvidenceView, JudgeSession } from "../../evidence";
+import type {
+  EarlyDecision,
+  JudgeRunInput,
+  OpenCodeStreamEvent,
+  SessionArchive,
+} from "../../types";
+import type { EvidenceQueryAudit } from "../evidence";
+import { credentialsFor, removeCredentials } from "../opencode/auth";
+import { createSessionDatabase, OPENCODE_VERSION } from "../opencode/host";
+import { runSession } from "../opencode/session";
+import { readArchivedSession } from "../opencode/archive";
+import { hash, writeJson } from "../files";
+import { installJudgeTools } from "./tools";
+import type { JudgeRequest, SubmissionAudit } from "./submission";
+import { judgeConfiguration, judgePrompt } from "./agent";
+
+/** Persistent native judge session. It can read evidence and decide, but cannot stop an eval. */
+export async function createJudgeSession(
+  input: Pick<
+    JudgeRunInput,
+    "model" | "rubric" | "timeoutMs" | "websearch" | "metrics" | "agent"
+  >,
+  directory: string,
+  onEvent: (event: OpenCodeStreamEvent) => void,
+): Promise<
+  JudgeSession & {
+    setCheck(id: string): void;
+    close(): Promise<SessionArchive | undefined>;
+  }
+> {
+  await mkdir(directory, { recursive: true });
+  const database = resolve(directory, "opencode.db");
+  await createSessionDatabase(
+    database,
+    credentialsFor(input.model, input.websearch),
+  );
+  let host: Awaited<ReturnType<typeof OpenCode.create>>;
+  try {
+    host = await OpenCode.create({
+      database: { path: database },
+      events: { persist: true },
+      config: await judgeConfiguration(input, directory),
+    });
+  } catch (error) {
+    removeCredentials(database);
+    throw error;
+  }
+  const queries: EvidenceQueryAudit[] = [];
+  const submissions: SubmissionAudit[] = [];
+  let checkId: string | undefined;
+  let rootId: string | undefined,
+    after: number | undefined,
+    busy = false,
+    closed = false;
+  const recorded = new Set<string>();
+  let tools: Awaited<ReturnType<typeof installJudgeTools>>;
+  try {
+    tools = await installJudgeTools(
+      host,
+      directory,
+      input,
+      queries,
+      submissions,
+    );
+  } catch (error) {
+    await host.close();
+    removeCredentials(database);
+    throw error;
+  }
+  const invoke = async (
+    evidence: EvidenceView,
+    signal: AbortSignal,
+    purpose: "early" | "final",
+  ) => {
+    signal.throwIfAborted();
+    if (busy || closed) throw new Error("Judge session is busy or closed");
+    busy = true;
+    let request: JudgeRequest | undefined;
+    try {
+      request = await tools.open(evidence, purpose, signal, checkId);
+      const result = await runSession(
+        host,
+        {
+          model: input.model,
+          agent: input.agent.id,
+          directory,
+          timeoutMs: input.timeoutMs,
+          signal,
+          sessionId: rootId,
+          after,
+          prompt: judgePrompt(purpose),
+        },
+        (event) => {
+          if (event.type === "session.created" && !event.data.parentID)
+            rootId ??= event.data.sessionID;
+          if ("durable" in event && event.durable.aggregateID === rootId)
+            after = Math.max(after ?? -1, event.durable.seq);
+          if (!("id" in event) || recorded.has(event.id)) return;
+          recorded.add(event.id);
+          onEvent(event);
+        },
+      );
+      rootId ??= result.sessions[0]?.info.id;
+      if (result.state !== "completed") {
+        signal.throwIfAborted();
+        throw new Error(result.error ?? `Judge ${result.state}`);
+      }
+      return request.result();
+    } finally {
+      request?.close();
+      busy = false;
+      await writeJson(resolve(directory, "evidence-queries.json"), queries);
+      await writeJson(
+        resolve(directory, "judgment-submissions.json"),
+        submissions,
+      );
+    }
+  };
+  return {
+    setCheck: (id) => {
+      checkId = id;
+    },
+    check: async (evidence, signal) => {
+      const decision = await invoke(evidence, signal, "early");
+      if (decision.kind === "continue") return decision;
+      if (decision.judgment.value === null)
+        throw new Error("Early judgments must be fully decided");
+      return {
+        ...decision,
+        judgment: { ...decision.judgment, value: decision.judgment.value },
+      } satisfies EarlyDecision;
+    },
+    grade: async (evidence, signal) => {
+      const decision = await invoke(evidence, signal, "final");
+      if (decision.kind !== "decided")
+        throw new Error("Final grading requires a judgment submission");
+      return decision.judgment;
+    },
+    close: async () => {
+      if (closed) throw new Error("Judge archive already finalized");
+      closed = true;
+      try {
+        await host.close();
+      } finally {
+        removeCredentials(database);
+      }
+      await writeJson(resolve(directory, "evidence-queries.json"), queries);
+      await writeJson(
+        resolve(directory, "judgment-submissions.json"),
+        submissions,
+      );
+      if (!rootId) return undefined;
+      const { result } = await readArchivedSession(database, rootId, () => {});
+      await writeJson(resolve(directory, "session.json"), result.sessions);
+      return {
+        sessionId: rootId,
+        database: "opencode.db",
+        databaseHash: hash(await Bun.file(database).bytes()),
+        opencodeVersion: OPENCODE_VERSION,
+        accounting: result.accounting,
+      };
+    },
+  };
+}

package/src/infra/judging/submission.ts

@@ -0,0 +1,138 @@
+import { randomUUID } from "node:crypto";
+import type { EvidenceView } from "../../evidence";
+import type { JudgeCheck, MetricDefinition } from "../../types";
+import { errorMessage } from "../files";
+import { validateCitations, validateMetrics } from "./contract";
+
+export type JudgePhase = "early" | "final";
+export type JudgeDecision = NonNullable<JudgeCheck["decision"]>;
+export type SubmissionAudit = {
+  requestId: string;
+  phase: JudgePhase;
+  checkId?: string;
+  checkpoint: EvidenceView["checkpoint"];
+  tool: "submit_judgment" | "continue_judging";
+  input: unknown;
+  startedAt: string;
+  completedAt?: string;
+  result?: JudgeDecision;
+  error?: string;
+};
+
+/** One submission channel per fixed view. Never parse the model's final text. */
+export class JudgeRequest {
+  readonly id = randomUUID();
+  closed = false;
+  private submitting = false;
+  private accepted?: JudgeDecision;
+
+  constructor(
+    readonly phase: JudgePhase,
+    readonly evidence: EvidenceView,
+    readonly metrics: readonly MetricDefinition[],
+    private readonly audit: SubmissionAudit[],
+    readonly checkId?: string,
+    private readonly signal?: AbortSignal,
+  ) {}
+
+  assertActive() {
+    this.signal?.throwIfAborted();
+    if (this.closed)
+      throw new Error(
+        "This judge request has closed; its evidence is no longer active",
+      );
+  }
+
+  context() {
+    this.assertActive();
+    return {
+      requestId: this.id,
+      phase: this.phase,
+      metrics: this.metrics,
+      evidence: this.evidence.summary(),
+    };
+  }
+
+  submit(input: { requestId: string; metrics: unknown }) {
+    return this.accept("submit_judgment", input, async () => {
+      const judgment = validateMetrics(input.metrics, this.metrics);
+      if (this.phase === "early" && judgment.value === null)
+        throw new Error(
+          "Early checks require every metric to be decided. Use continue_judging when evidence is insufficient",
+        );
+      await validateCitations(judgment, this.evidence);
+      return { kind: "decided", judgment };
+    });
+  }
+
+  continue(input: { requestId: string; reason: string }) {
+    return this.accept("continue_judging", input, async () => {
+      if (this.phase !== "early")
+        throw new Error(
+          "Final grading requires submit_judgment; unresolved metrics can be null",
+        );
+      if (typeof input.reason !== "string" || !input.reason.trim())
+        throw new Error("Explain what evidence is still needed");
+      return { kind: "continue", reason: input.reason };
+    });
+  }
+
+  result(): JudgeDecision {
+    this.assertActive();
+    if (!this.accepted)
+      throw new Error(
+        "Judge ended without an accepted submission through submit_judgment or continue_judging",
+      );
+    return structuredClone(this.accepted);
+  }
+
+  close() {
+    this.closed = true;
+  }
+
+  private async accept(
+    tool: SubmissionAudit["tool"],
+    input: { requestId: string },
+    work: () => Promise<JudgeDecision>,
+  ): Promise<{ accepted: true }> {
+    const entry: SubmissionAudit = {
+      requestId: this.id,
+      phase: this.phase,
+      checkId: this.checkId,
+      checkpoint: this.evidence.checkpoint,
+      tool,
+      input: structuredClone(input),
+      startedAt: new Date().toISOString(),
+    };
+    this.audit.push(entry);
+    let acquired = false;
+    try {
+      this.assertActive();
+      if (input.requestId !== this.id)
+        throw new Error(
+          "Stale requestId; read judge_context for the current request",
+        );
+      if (this.accepted)
+        throw new Error(
+          "A result is already accepted for this request; finish the turn",
+        );
+      if (this.submitting)
+        throw new Error(
+          "Another submission is being validated; await it before submitting again",
+        );
+      this.submitting = acquired = true;
+      const decision = await work();
+      // Citation reads can outlive cancellation. They must not commit into a later check.
+      this.assertActive();
+      this.accepted = structuredClone(decision);
+      entry.result = structuredClone(decision);
+      return { accepted: true };
+    } catch (error) {
+      entry.error = errorMessage(error);
+      throw error;
+    } finally {
+      if (acquired) this.submitting = false;
+      entry.completedAt = new Date().toISOString();
+    }
+  }
+}

package/src/infra/judging/tools.ts

@@ -0,0 +1,145 @@
+import type { OpenCode } from "@opencode/sdk";
+import { Schema } from "effect";
+import type { EvidenceView } from "../../evidence";
+import type { JudgeRunInput } from "../../types";
+import type { EvidenceQueryAudit } from "../evidence";
+import { JUDGE_TOOLS } from "./agent";
+import { evidenceTool, toolResult } from "./evidence-tool";
+import { metricsSchema, STRICT_OBJECT } from "./contract";
+import {
+  JudgeRequest,
+  type JudgePhase,
+  type SubmissionAudit,
+} from "./submission";
+
+/** Tools capture a fixed request at registry reload, including its evidence view.
+ * https://opencode.ai/v2/docs/build/plugins/#tools
+ */
+export async function installJudgeTools(
+  host: OpenCode.Interface,
+  directory: string,
+  input: Pick<JudgeRunInput, "metrics" | "websearch">,
+  queries: EvidenceQueryAudit[],
+  submissions: SubmissionAudit[],
+) {
+  const metrics = input.metrics;
+  if (!metrics.length) throw new Error("Declare at least one judge metric");
+  const allowed = new Set(
+    JUDGE_TOOLS.filter(
+      (tool) => input.websearch || !["websearch", "webfetch"].includes(tool),
+    ),
+  );
+  let active: JudgeRequest | undefined,
+    reload: (() => Promise<void>) | undefined;
+  await host.plugin({
+    id: "judge-tools",
+    async setup(context) {
+      reload = () => context.tool.reload();
+      await context.tool.transform((tools) => {
+        for (const tool of tools.list()) {
+          if (!allowed.has(tool.id)) tools.remove(tool.id);
+          else
+            tools.update(tool.id, (tool) => {
+              tool.options = { ...tool.options, codemode: true, pinned: true };
+            });
+        }
+        const request = active;
+        if (!request || request.closed) return;
+        const options = { codemode: true as const, pinned: true };
+        // Keep catalog shapes stable across a persistent session. The current
+        // phase and ID are data from judge_context, checked by the request owner.
+        const requestId = Schema.NonEmptyString;
+        const receipt = Schema.Struct({ accepted: Schema.Literal(true) });
+        tools.add(
+          evidenceTool(request.evidence, queries, {
+            active: () => request.assertActive(),
+            audit: () => ({
+              checkId: request.checkId,
+              checkpoint: request.evidence.checkpoint,
+            }),
+          }),
+        );
+        tools.add({
+          name: "judge_context",
+          options,
+          description:
+            "Get this request's ID, phase, declared metrics, and current evidence index as structured data. Read this at the start of each judge turn.",
+          input: Schema.Struct({}).annotate(STRICT_OBJECT),
+          output: Schema.Struct({
+            requestId: Schema.String,
+            phase: Schema.Literals(["early", "final"]),
+            metrics: Schema.Array(
+              Schema.Struct({ id: Schema.String, name: Schema.String }),
+            ),
+            evidence: Schema.Record(Schema.String, Schema.Unknown),
+          }),
+          async execute() {
+            return toolResult(request.context());
+          },
+        });
+        tools.add({
+          name: "submit_judgment",
+          options,
+          description:
+            "Submit all metric decisions for this fixed evidence view. Values, required metric IDs, citations, and exact quotes are validated before acceptance. Errors are returned to you; correct the arguments and call again. An early submission must be irreversible for every metric. After acceptance, finish your turn.",
+          input: Schema.Struct({
+            requestId,
+            metrics: metricsSchema(metrics),
+          }).annotate(STRICT_OBJECT),
+          output: receipt,
+          async execute(value) {
+            return toolResult(await request.submit(value));
+          },
+        });
+        tools.add({
+          name: "continue_judging",
+          options,
+          description:
+            "Finish this early check when more evidence is needed. Explain what is missing. This does not stop or steer the candidate. After acceptance, finish your turn.",
+          input: Schema.Struct({
+            requestId,
+            reason: Schema.NonEmptyString,
+          }).annotate(STRICT_OBJECT),
+          output: receipt,
+          async execute(value) {
+            return toolResult(await request.continue(value));
+          },
+        });
+      });
+      await context.session.hook("context", (event) => {
+        for (const name of Object.keys(event.tools))
+          if (name !== "execute" && !allowed.has(name))
+            delete event.tools[name];
+      });
+    },
+  });
+  await host.plugin.awaitActivation({ location: { directory } });
+  if (!reload) throw new Error("Judge tools did not activate");
+  return {
+    async open(
+      evidence: EvidenceView,
+      phase: JudgePhase,
+      signal?: AbortSignal,
+      checkId?: string,
+    ) {
+      if (active && !active.closed)
+        throw new Error("Judge request is still active");
+      const request = new JudgeRequest(
+        phase,
+        evidence,
+        metrics,
+        submissions,
+        checkId,
+        signal,
+      );
+      active = request;
+      try {
+        await reload!();
+        return request;
+      } catch (error) {
+        request.close();
+        throw error;
+      }
+    },
+  };
+}

package/src/infra/opencode/archive.ts

@@ -0,0 +1,81 @@
+import { OpenCode } from "@opencode/sdk";
+import { dirname, resolve } from "node:path";
+import type { SessionInfo } from "@opencode/client";
+import type { OpenCodeStreamEvent, ToolCall } from "../../types";
+import {
+  lastResponse,
+  trackTool,
+  type RecordedSession,
+  type SessionResult,
+} from "./session";
+
+/** Read the stopped database through native APIs, with no prompts or live transport. */
+export async function readArchivedSession(
+  database: string,
+  rootId: string,
+  onEvent: (event: OpenCodeStreamEvent) => void,
+) {
+  await using host = await OpenCode.create({
+    database: { path: database },
+    config: {
+      directory: resolve(dirname(database), "archive-configuration"),
+      project: false,
+      content: JSON.stringify({ websearch: false }),
+    },
+  });
+  const sessions: RecordedSession[] = [];
+  const ids = new Set([rootId]);
+  const tools = new Map<string, ToolCall>();
+  let outcome: SessionInfo["outcome"], error: string | undefined;
+  for (const id of ids) {
+    if (id === rootId) {
+      let cursor: string | undefined;
+      do {
+        const children = await host.session.list({ limit: 100, cursor });
+        for (const child of children.data) ids.add(child.id);
+        cursor = children.cursor.next ?? undefined;
+      } while (cursor);
+    }
+    for await (const event of host.session.log({
+      sessionID: id,
+      follow: false,
+    })) {
+      onEvent(event);
+      if (event.type.startsWith("session.tool.")) trackTool(tools, event);
+      if (id !== rootId) continue;
+      if (event.type === "session.execution.succeeded") outcome = "succeeded";
+      if (event.type === "session.execution.interrupted")
+        outcome = "interrupted";
+      if (event.type === "session.execution.failed") {
+        outcome = "failed";
+        error = event.data.error.message;
+      }
+    }
+    sessions.push({
+      info: await host.session.get({ sessionID: id }),
+      messages: await host.session.context({ sessionID: id }),
+    });
+  }
+  outcome ??= sessions[0].info.outcome;
+  const stats = await host.session.stats({ tools: "none" });
+  const result: SessionResult = {
+    sessions,
+    text: lastResponse(sessions[0].messages),
+    tools: [...tools.values()],
+    state: outcome === "succeeded" ? "completed" : "failed",
+    error:
+      outcome === "succeeded"
+        ? undefined
+        : (error ??
+          (outcome === "interrupted"
+            ? "Session interrupted"
+            : "Session did not complete")),
+    accounting: {
+      costUSD: stats.cost,
+      sessions: stats.sessions,
+      steps: stats.steps,
+      tokens: stats.tokens,
+    },
+  };
+  return { result, outcome };
+}

package/src/infra/opencode/auth.ts

@@ -0,0 +1,119 @@
+import { Database } from "@opencode/core/database/database";
+import { Global } from "@opencode/util/global";
+import { makeGlobalNode } from "@opencode/util/effect/app-node";
+import { Context, Effect, Layer } from "effect";
+import { Database as SQLite } from "bun:sqlite";
+import { isAbsolute, join } from "node:path";
+import type { ModelRef } from "../../types";
+
+export type Credential = {
+  id: string;
+  integration_id: string | null;
+  label: string;
+  value: string;
+  connector_id: string | null;
+  method_id: string | null;
+  active: number | null;
+  time_created: number;
+  time_updated: number;
+};
+export const localCredentialPath = () => {
+  const path = process.env.OPENCODE_DB ?? "opencode.db";
+  return isAbsolute(path) ? path : join(Global.Path.data, path);
+};
+export function createCredentialSnapshot(
+  source = localCredentialPath(),
+  integrations?: readonly string[],
+): Credential[] {
+  const db = new SQLite(source, { readonly: true });
+  try {
+    const rows = db
+      .query(
+        "SELECT id,integration_id,label,value,connector_id,method_id,active,time_created,time_updated FROM credential",
+      )
+      .all() as Credential[];
+    return rows
+      .filter(
+        (row) =>
+          row.active !== 0 &&
+          (!integrations || integrations.includes(row.integration_id ?? "")),
+      )
+      .map((row) => {
+        const parsed = JSON.parse(row.value) as {
+          type: string;
+          refresh?: string;
+        };
+        return {
+          ...row,
+          value:
+            parsed.type === "oauth"
+              ? JSON.stringify({ ...parsed, refresh: "" })
+              : row.value,
+        };
+      });
+  } finally {
+    db.close(true);
+  }
+}
+export const credentialsFor = (model: ModelRef, websearch: "exa" | false) => {
+  const provider = model.split("/")[0];
+  const integration = provider.startsWith("console-") ? "opencode" : provider;
+  const required = [
+    ...new Set([integration, ...(websearch ? ["opencode", "exa"] : [])]),
+  ];
+  const credentials = createCredentialSnapshot(undefined, required);
+  if (!credentials.some((row) => row.integration_id === integration))
+    throw new Error(`No active OpenCode connection for ${integration}`);
+  return credentials;
+};
+export const databaseWithCredentials = (
+  path: string,
+  credentials: Credential[],
+) =>
+  makeGlobalNode({
+    service: Database.Service,
+    layer: Database.layer({ path }).pipe(
+      Layer.tap((context) => {
+        const database = Context.get(context, Database.Service);
+        return Effect.gen(function* () {
+          for (const row of credentials) {
+            const values = [
+              row.id,
+              row.integration_id,
+              row.label,
+              row.value,
+              row.connector_id,
+              row.method_id,
+              row.active,
+              row.time_created,
+              row.time_updated,
+            ].map((value) =>
+              value === null
+                ? "NULL"
+                : typeof value === "number"
+                  ? String(value)
+                  : `'${value.replaceAll("'", "''")}'`,
+            );
+            yield* database.db
+              .run(
+                `INSERT INTO credential (id,integration_id,label,value,connector_id,method_id,active,time_created,time_updated) VALUES (${values.join(",")})`,
+              )
+              .pipe(Effect.orDie);
+          }
+        });
+      }),
+    ),
+    deps: [Global.node],
+  });
+
+/** Purge secrets before the database becomes a finalized evidence artifact. */
+export function removeCredentials(path: string) {
+  const db = new SQLite(path);
+  try {
+    db.exec(
+      "PRAGMA busy_timeout=5000; PRAGMA wal_checkpoint(TRUNCATE); PRAGMA secure_delete=ON; DELETE FROM credential; VACUUM; PRAGMA wal_checkpoint(TRUNCATE);",
+    );
+  } finally {
+    db.close(true);
+  }
+}