@hona/openeval 0.2.0

package/src/judgment.ts

@@ -0,0 +1,40 @@
+import type { MetricDefinition, MetricJudgment } from "./types";
+
+export const JUDGE_PROTOCOL = 3;
+export const isScored = (value: unknown): value is number =>
+  typeof value === "number" &&
+  Number.isFinite(value) &&
+  value >= 0 &&
+  value <= 1;
+
+/** Every rubric explicitly declares its metrics. */
+export function rubricMetrics(rubric: string): MetricDefinition[] {
+  const metrics = [
+    ...rubric.matchAll(/^## Metric: ([a-z][a-z0-9_]*)[ \t]*[—–-][ \t]*(.+)$/gm),
+  ].map((match) => ({
+    id: match[1],
+    name: match[2].trim(),
+  }));
+  if (new Set(metrics.map((metric) => metric.id)).size !== metrics.length)
+    throw new Error("Metric IDs must be unique within judge.md");
+  if (
+    !metrics.length ||
+    metrics.some((metric) => !metric.name) ||
+    metrics.length !== (rubric.match(/^## Metric:/gm)?.length ?? 0)
+  )
+    throw new Error(
+      "Declare every metric as ## Metric: id — Label in judge.md",
+    );
+  return metrics;
+}
+
+/** Equal weighting within an eval; no task-specific policy is implemented here.
+ * https://arxiv.org/html/2607.07946#S5.SS3 (equal task weighting)
+ */
+export function metricMean(
+  metrics: readonly Pick<MetricJudgment, "value">[],
+): number | null {
+  return metrics.length && metrics.every((metric) => isScored(metric.value))
+    ? metrics.reduce((sum, metric) => sum + metric.value!, 0) / metrics.length
+    : null;
+}

package/src/runtime.ts

@@ -0,0 +1,60 @@
+/** Recorded periods of work. An omitted end means the worker is still active. */
+export type RuntimeInterval = { start: number; end?: number };
+
+/** Wall-clock runtime is the union of work intervals, not worker-hours or calendar age.
+ * Resource/time methodology: https://www.anthropic.com/engineering/infrastructure-noise
+ */
+export function mergeRuntime(
+  intervals: readonly RuntimeInterval[],
+): RuntimeInterval[] {
+  const sorted = intervals
+    .filter(
+      (item) =>
+        Number.isFinite(item.start) &&
+        (item.end === undefined ||
+          (Number.isFinite(item.end) && item.end >= item.start)),
+    )
+    .map((item) => ({ ...item }))
+    .sort((a, b) => a.start - b.start);
+  const merged: RuntimeInterval[] = [];
+  for (const interval of sorted) {
+    const previous = merged.at(-1);
+    if (!previous || interval.start > (previous.end ?? Infinity))
+      merged.push(interval);
+    else
+      previous.end =
+        previous.end === undefined || interval.end === undefined
+          ? undefined
+          : Math.max(previous.end, interval.end);
+  }
+  return merged;
+}
+
+export const runtimeMs = (intervals: readonly RuntimeInterval[], now: number) =>
+  mergeRuntime(intervals).reduce(
+    (sum, item) =>
+      sum + Math.max(0, Math.min(item.end ?? now, now) - item.start),
+    0,
+  );
+
+export const timestampMs = (value?: string) => {
+  const time = value === undefined ? NaN : Date.parse(value);
+  return Number.isFinite(time) ? time : undefined;
+};
+
+/** Freeze live counters at the last heartbeat when a runner stops reporting. */
+export function runtimeClock(
+  now: number,
+  heartbeat?: string | number,
+  running = true,
+) {
+  const observed =
+    typeof heartbeat === "number"
+      ? Number.isFinite(heartbeat)
+        ? heartbeat
+        : undefined
+      : timestampMs(heartbeat);
+  return observed !== undefined && (!running || now - observed > 30_000)
+    ? Math.min(now, observed)
+    : now;
+}

package/src/session-replay.ts

@@ -0,0 +1,245 @@
+import type {
+  JsonValue,
+  ModelRef,
+  SessionMessageAssistant,
+  SessionMessageAssistantTool,
+  SessionStructuredError,
+  ToolContent,
+} from "@opencode/client";
+import type { RunEvent } from "./types";
+import type { RecordedSession, SessionStage } from "./view";
+export type { RecordedSession, SessionStage, SessionSnapshot } from "./view";
+
+export const emptyStage = (): SessionStage => ({
+  prompt: "",
+  model: "",
+  sessions: [],
+});
+export const sessionModel = (ref: string): ModelRef => {
+  const [model, variant] = ref.split("#"),
+    slash = model.indexOf("/");
+  return {
+    providerID: slash < 0 ? "recorded" : model.slice(0, slash),
+    id: slash < 0 ? model : model.slice(slash + 1),
+    variant,
+  };
+};
+const object = (value: unknown): Record<string, JsonValue> =>
+  value && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, JsonValue>)
+    : {};
+const toolContent = (value: unknown): [ToolContent, ...ToolContent[]] => {
+  const items = (Array.isArray(value) ? value : []).filter(
+    (item): item is ToolContent =>
+      !!item &&
+      typeof item === "object" &&
+      ((item.type === "text" && typeof item.text === "string") ||
+        (item.type === "file" &&
+          typeof item.uri === "string" &&
+          typeof item.mime === "string")),
+  );
+  return items.length
+    ? (items as [ToolContent, ...ToolContent[]])
+    : [
+        {
+          type: "text",
+          text: typeof value === "string" ? value : JSON.stringify(value ?? ""),
+        },
+      ];
+};
+const structuredError = (value: unknown): SessionStructuredError => {
+  const data = object(value);
+  return {
+    type: typeof data.type === "string" ? data.type : "RecordedError",
+    message:
+      typeof data.message === "string"
+        ? data.message
+        : String(value ?? "Unknown error"),
+  };
+};
+
+/** Incremental native event projection. Completed views use the authoritative session context. */
+export function applySessionRecord(stage: SessionStage, record: RunEvent) {
+  if (!("data" in record.event)) return;
+  const event = record.event,
+    data = event.data as Record<string, unknown>,
+    time = Date.parse(record.time);
+  if (typeof data.sessionID !== "string") return;
+  let session = stage.sessions.find(
+    (session) => session.sessionID === data.sessionID,
+  );
+  if (!session) {
+    session = {
+      sessionID: data.sessionID,
+      title: "",
+      created: time,
+      model: sessionModel(stage.model),
+      agent: "build",
+      messages: [],
+      diffs: [],
+      status: { type: "idle" },
+      parts: {},
+    };
+    stage.sessions.push(session);
+  }
+  if (event.type === "session.created") {
+    if (typeof data.parentID === "string") session.parentID = data.parentID;
+    if (typeof data.agent === "string") session.agent = data.agent;
+    if (typeof data.title === "string") session.title = data.title;
+    if (data.model && typeof data.model === "object")
+      session.model = data.model as ModelRef;
+  }
+  if (event.type === "session.renamed" && typeof data.title === "string")
+    session.title = data.title;
+  if (event.type === "session.execution.started")
+    session.status = { type: "busy" };
+  if (
+    [
+      "session.execution.succeeded",
+      "session.execution.failed",
+      "session.execution.interrupted",
+    ].includes(event.type)
+  )
+    session.status = { type: "idle" };
+  if (event.type === "session.inbox.enqueued") {
+    const item = data.item as
+      { type?: string; payload?: { text?: string } } | undefined;
+    if (
+      item?.type === "user" &&
+      typeof item.payload?.text === "string" &&
+      !session.messages.some((message) => message.id === data.inboxID)
+    )
+      session.messages.push({
+        type: "user",
+        id: String(data.inboxID),
+        text: item.payload.text,
+        time: { created: time },
+      });
+  }
+  if (typeof data.assistantMessageID !== "string") return;
+  let message = session.messages.find(
+    (message): message is SessionMessageAssistant =>
+      message.type === "assistant" && message.id === data.assistantMessageID,
+  );
+  if (!message) {
+    message = {
+      type: "assistant",
+      id: data.assistantMessageID,
+      time: { created: time },
+      agent: session.agent,
+      model: session.model,
+      content: [],
+    };
+    session.messages.push(message);
+  }
+  if (event.type === "session.step.started") {
+    message.agent = typeof data.agent === "string" ? data.agent : session.agent;
+    message.model =
+      data.model && typeof data.model === "object"
+        ? (data.model as ModelRef)
+        : session.model;
+  }
+  if (event.type === "session.step.streamed") message.time.streamed = time;
+  if (event.type === "session.step.ended") {
+    message.time.completed = time;
+    message.finish = data.finish as SessionMessageAssistant["finish"];
+  }
+  const text = /^session\.(text|reasoning)\.(started|delta|ended)$/.exec(
+    event.type,
+  );
+  if (text) {
+    const type = text[1] as "text" | "reasoning",
+      key = `${message.id}:${type}:${data.ordinal}`;
+    let part = session.parts[key];
+    if (!part) {
+      part = {
+        messageID: message.id,
+        index: message.content.length,
+        complete: false,
+      };
+      session.parts[key] = part;
+      message.content.push(
+        type === "reasoning"
+          ? { type, text: "", time: { created: time } }
+          : { type, text: "" },
+      );
+    }
+    const content = message.content[part.index];
+    if (content.type === "tool") return;
+    if (text[2] === "delta" && !part.complete)
+      content.text += String(data.delta ?? "");
+    if (text[2] === "ended") {
+      content.text = String(data.text ?? content.text);
+      part.complete = true;
+      if (content.type === "reasoning")
+        content.time = {
+          created: content.time?.created ?? time,
+          completed: time,
+        };
+    }
+  }
+  if (!event.type.startsWith("session.tool.") || typeof data.id !== "string")
+    return;
+  let tool = message.content.find(
+    (part): part is SessionMessageAssistantTool =>
+      part.type === "tool" && part.id === data.id,
+  );
+  if (!tool) {
+    tool = {
+      type: "tool",
+      id: data.id,
+      name: String(data.name ?? "tool"),
+      time: { created: time },
+      state: { status: "streaming", input: "" },
+    };
+    message.content.push(tool);
+  }
+  if (typeof data.name === "string") tool.name = data.name;
+  if (
+    event.type === "session.tool.input.delta" &&
+    tool.state.status === "streaming"
+  )
+    tool.state.input += String(data.delta ?? "");
+  if (
+    event.type === "session.tool.input.ended" &&
+    tool.state.status === "streaming"
+  )
+    tool.state.input = String(data.text ?? "");
+  if (event.type === "session.tool.called") {
+    tool.state = { status: "running", input: object(data.input), metadata: {} };
+    tool.time.ran = time;
+  }
+  if (
+    event.type === "session.tool.progress" &&
+    tool.state.status !== "streaming"
+  )
+    tool.state.metadata = object(data.metadata);
+  if (
+    event.type === "session.tool.success" ||
+    event.type === "session.tool.failed"
+  ) {
+    const input = tool.state.status === "streaming" ? {} : tool.state.input,
+      metadata = data.metadata
+        ? object(data.metadata)
+        : tool.state.status === "streaming"
+          ? {}
+          : tool.state.metadata;
+    tool.state =
+      event.type === "session.tool.success"
+        ? {
+            status: "completed",
+            input,
+            metadata,
+            content: toolContent(data.content),
+          }
+        : {
+            status: "error",
+            input,
+            metadata,
+            error: structuredError(data.error),
+            ...(data.content ? { content: toolContent(data.content) } : {}),
+          };
+    tool.time.completed = time;
+  }
+  if (typeof data.executed === "boolean") tool.executed = data.executed;
+}

package/src/types.ts

@@ -0,0 +1,302 @@
+import type {
+  OpenCodeEvent,
+  SessionLogItem,
+  ConfigEntry,
+  AgentInfo,
+} from "@opencode/client";
+export type OpenCodeStreamEvent = OpenCodeEvent | SessionLogItem;
+
+export const CANDIDATE_TIMEOUT_MS = 45 * 60 * 1000;
+export type ModelRef = `${string}/${string}`;
+export type Engine = "docker" | "podman";
+export type ProviderDefinitions = NonNullable<
+  Extract<ConfigEntry, { type: "document" }>["info"]["providers"]
+>;
+export type SessionState =
+  "running" | "completed" | "stopped" | "failed" | "timed_out";
+export type Stage = "candidate" | "judge";
+
+export type Judge = {
+  model: ModelRef;
+  timeoutMs?: number;
+  websearch?: "exa" | false;
+};
+export type Benchmark = {
+  name?: string;
+  models: readonly ModelRef[];
+  judge: Judge;
+  repetitions?: number;
+  concurrency?: number;
+  candidate?: {
+    timeoutMs?: number;
+    websearch?: "exa" | false;
+    providers?: ProviderDefinitions;
+  };
+  container?: {
+    engine?: Engine;
+    image?: string;
+    cpus?: number;
+    memoryMiB?: number;
+  };
+};
+export type PreparationStep = { cwd: string; argv: readonly string[] };
+export type MonitorPolicy = {
+  minIntervalMs: number;
+  maxChecks: number;
+  maxCostUSD: number;
+};
+export type Eval = {
+  /** Let the host judge stop execution once the criterion is conclusively decided. */
+  earlyStop?:
+    boolean | (Partial<MonitorPolicy> & { onlyModels?: readonly ModelRef[] });
+  workspace?: {
+    repository?: string;
+    /** Readable file overlays committed in order to the named refs, on the host. */
+    revisions?: readonly { ref: string; directory: string; message: string }[];
+    ref: string;
+    commit?: string;
+    checkout?: string;
+    remote?: string;
+    overlay?: string;
+  };
+  prepare?: readonly PreparationStep[];
+};
+export type EvalDefinition = {
+  id: string;
+  directory: string;
+  prompt: string;
+  judge: string;
+  settings: Eval;
+  sourceHash: string;
+  judgeHash: string;
+  /** Declared by `## Metric: id — Label` headings in judge.md. */
+  metrics: MetricDefinition[];
+  name: string;
+};
+export type BenchmarkDefinition = {
+  name: string;
+  directory: string;
+  models: ModelRef[];
+  judge: Required<Judge>;
+  repetitions: number;
+  concurrency: number;
+  candidate: {
+    timeoutMs: number;
+    websearch: "exa" | false;
+    providers?: ProviderDefinitions;
+  };
+  container: { engine: Engine; image: string; cpus: number; memoryMiB: number };
+  evals: EvalDefinition[];
+};
+export type Cost = {
+  usd: number | null;
+  reportedUSD: number;
+  complete: boolean;
+};
+export type Accounting = {
+  costUSD: number;
+  sessions: number;
+  steps: number;
+  tokens: {
+    input: number;
+    output: number;
+    reasoning: number;
+    cache: { read: number; write: number };
+  };
+};
+export type SessionArchive = {
+  sessionId: string;
+  database: string;
+  databaseHash: string;
+  opencodeVersion: string;
+  accounting?: Accounting;
+};
+export type ToolCall = {
+  id: string;
+  name: string;
+  assistantMessageId: string;
+  status: "preparing" | "called" | "succeeded" | "failed";
+  input?: Record<string, unknown>;
+  rawInput?: string;
+  content?: unknown;
+  error?: unknown;
+  startedAt?: number;
+  completedAt?: number;
+  durationMs?: number;
+  executed?: boolean;
+};
+export type EvidenceRef = { directory: string; hash: string };
+export type EvidenceCheckpoint = EvidenceRef & {
+  /** Monotonic feed revision used to coalesce updates. */
+  revision: number;
+  /** Inclusive, zero-based sequence within this fixed evidence view. */
+  through: number;
+  createdAt: string;
+  cutoffAt?: number;
+  boundaryEventId?: string;
+  cursors?: Record<string, number>;
+};
+export type EarlyDecision =
+  | { kind: "continue"; reason: string }
+  | { kind: "decided"; judgment: Judgment & { value: number } };
+export type JudgeCheck = {
+  id: string;
+  judgeRunId: string;
+  purpose: "early" | "final";
+  checkpoint: EvidenceCheckpoint;
+  state: "running" | "completed" | "failed" | "cancelled";
+  startedAt: string;
+  /** Worker admission time, excluding time queued for judge capacity. */
+  executionStartedAt?: string;
+  completedAt?: string;
+  decision?:
+    | { kind: "continue"; reason: string }
+    | { kind: "decided"; judgment: Judgment };
+  error?: string;
+  /** Range in the runner log containing this judge turn. */
+  events: { after: number; through?: number };
+};
+export type EvalStop = {
+  reason: "judge_decided";
+  requestedAt: string;
+  judgeRunId: string;
+  checkId: string;
+  checkpoint: EvidenceCheckpoint;
+  applied?: boolean;
+};
+export type EvalRunInput = {
+  evalId: string;
+  model: ModelRef;
+  repetition: number;
+  prompt: string;
+  candidateHash: string;
+  sourceHash: string;
+  imageId: string;
+  timeoutMs: number;
+  earlyStop: boolean;
+  runtime: BenchmarkRun["runtime"];
+};
+export type EvalRun = {
+  id: string;
+  slotId: string;
+  input: EvalRunInput;
+  state: SessionState;
+  startedAt: string;
+  completedAt?: string;
+  elapsedMs?: number;
+  evidence?: EvidenceRef;
+  session?: SessionArchive;
+  error?: string;
+  replaces?: string;
+  stop?: EvalStop;
+};
+export type MetricDefinition = { id: string; name: string };
+export type EvidenceCitation = (
+  | { kind: "response" }
+  | { kind: "tool" | "message" | "event"; id: string }
+  | { kind: "artifact"; path: string; revision: "initial" | "final" }
+) & { quote?: string; offset?: number };
+export type MetricJudgment = {
+  id: string;
+  value: 0 | 1 | null;
+  reason: string;
+  evidence: EvidenceCitation[];
+  sources?: string[];
+};
+/** value is the equal-weight metric mean, or null if any metric is unknown. */
+export type Judgment = {
+  value: number | null;
+  reason: string;
+  metrics: MetricJudgment[];
+};
+export type JudgeRunInput = {
+  evalRunId: string;
+  evidence: EvidenceRef;
+  rubric: string;
+  /** Shared native agent profile used for this judgment. */
+  agent: Pick<AgentInfo, "id" | "mode" | "description" | "permissions"> & {
+    system: string;
+  };
+  model: ModelRef;
+  judgeHash: string;
+  timeoutMs: number;
+  websearch: "exa" | false;
+  mode: "monitor" | "final";
+  runtimeHash: string;
+  metrics: MetricDefinition[];
+  protocol: number;
+  monitor?: MonitorPolicy;
+};
+export type JudgeRun = {
+  id: string;
+  input: JudgeRunInput;
+  state: SessionState;
+  startedAt: string;
+  completedAt?: string;
+  elapsedMs?: number;
+  archiveStartedAt?: string;
+  judgment?: Judgment;
+  session?: SessionArchive;
+  error?: string;
+  activity?: "watching" | "queued" | "checking" | "finalizing";
+  decisionCheckId?: string;
+  monitorError?: string;
+  monitorErrorAt?: string;
+  monitorLimit?: string;
+};
+export type Slot = {
+  id: string;
+  evalId: string;
+  model: ModelRef;
+  repetition: number;
+  candidateHash: string;
+  judgeHash: string;
+  /** Predecessor while this slot awaits an execution with changed inputs. */
+  previousEvalRunId?: string;
+  evalRunId: string | null;
+  judgeRunId: string | null;
+  active: boolean;
+};
+export type BenchmarkRun = {
+  id: string;
+  name: string;
+  source: string;
+  createdAt: string;
+  updatedAt: string;
+  completedAt?: string;
+  state: "running" | "completed" | "incomplete" | "failed";
+  /** Work selected by the current run/retry/rejudge operation. */
+  scheduledSlotIds: readonly string[];
+  definition: BenchmarkDefinition;
+  runtime: { imageId: string; candidateHash: string; judgeHash: string };
+  error?: string;
+  mergedInto?: string;
+  sources?: Array<{
+    id: string;
+    directory: string;
+    mergedAt: string;
+    databaseHash: string;
+  }>;
+  execution: {
+    startedAt: string;
+    onlyModels?: readonly ModelRef[];
+    onlyEvals?: readonly string[];
+    onlyRepetitions?: readonly number[];
+    budgetUSD?: number;
+    estimatedUSD: number | null;
+    spentUSD: number;
+    deferred: number;
+  };
+};
+export type PlanItem = {
+  slot: Slot;
+  action: "candidate" | "judge" | "reuse" | "failed" | "deferred";
+  reason: string;
+};
+export type RunEvent = {
+  stage: Stage;
+  executionId: string;
+  time: string;
+  event: OpenCodeStreamEvent;
+};
+export type ExecutionObserver = (event: RunEvent) => void;