@visual-z/xeno 0.7.2 → 0.8.1

package/src/tui/ui/status.ts

@@ -14,6 +14,7 @@
  */
 import { clipSegs, clipText, segsWidth, splitLine, type Label, type Seg } from "./edge.ts";
 import { displayWidth } from "../width.ts";
+import { formatElapsed } from "./duration.ts";
 
 export interface ContextStatus {
   readonly used: number;
@@ -30,7 +31,13 @@ export interface StatusModel {
   readonly version: string;
   /** `/name`, else the first prompt; null for a session with neither. */
   readonly title: string | null;
-  readonly goal?: { readonly text: string; readonly status: string; readonly execution: string; readonly observations: number; readonly retries: number };
+  /**
+   * An explicit /goal, prepared by apps from the log (core/goal-view.ts). The
+   * block exists only while a goal is set, and for one message after it is met.
+   */
+  readonly goal?: GoalStatusView;
+  /** Without a goal: the latest check after this request's last edit, when it changed files. */
+  readonly checks?: { readonly status: "unchecked" | "pass" | "fail" | "stale" | "partial"; readonly command?: string; readonly exit?: number };
   readonly workspace: string;
   readonly workspaceShort: string;
   readonly branch: string | null;
@@ -39,7 +46,8 @@ export interface StatusModel {
   readonly activity?: { readonly state: string; readonly label: string; readonly detail: string; readonly queued: number };
   readonly git?: { readonly staged: number; readonly changed: number; readonly untracked: number; readonly conflicted: number; readonly ahead: number; readonly behind: number } | null;
   readonly context?: ContextStatus;
-  readonly usage?: { readonly input: number; readonly output: number; readonly cacheRate: number; readonly cost: string | null };
+  /** `speed`: output tokens per second over this turn's model requests (runtime/usage.ts generationSpeed). */
+  readonly usage?: { readonly input: number; readonly output: number; readonly cacheRate: number; readonly cost: string | null; readonly speed?: number };
   readonly tools: { readonly running: number; readonly done: number; readonly failed: number };
   readonly handoff?: { readonly verified: readonly string[]; readonly inFlight: readonly string[]; readonly unverified: readonly string[] };
   readonly tasks?: readonly { readonly what: string; readonly status: "active" | "blocked" }[];
@@ -49,6 +57,18 @@ export interface StatusModel {
   readonly extension?: string;
 }
 
+export interface GoalStatusView {
+  readonly phase: "drafting" | "working" | "needs_you" | "met" | "stopped";
+  readonly objective: string;
+  readonly elapsedMs: number;
+  readonly checks: readonly { readonly id: string; readonly kind: "command" | "human" | "fresh_check"; readonly status: string; readonly guard?: true; readonly exit?: number }[];
+  /** S.1: one glyph per rung (ran, tests the change, independent, human), prepared by apps. */
+  readonly ladder?: { readonly glyphs: string; readonly summary: string };
+  /** Model requests since the goal was set (from the log) and output tokens this process saw. */
+  readonly requests: number;
+  readonly tokens?: number;
+}
+
 export type Line = readonly Seg[];
 
 /** 1234 → "1.2k", 1234567 → "1.2m". The same compact form everywhere a token count appears. */
@@ -148,6 +168,10 @@ interface Block {
   readonly items: readonly Line[];
   /** A list block with no items is not shown at all. */
   readonly list: boolean;
+  /** Rows shown after the items, kept with the head (the goal's ladder and spend). */
+  readonly tail?: readonly Line[];
+  /** A list shown even without items: a goal being drafted has no checks yet. */
+  readonly keep?: boolean;
 }
 
 /**
@@ -158,12 +182,12 @@ interface Block {
  */
 export function sidebarLines(model: StatusModel, width: number, height: number): Line[] {
   const w = Math.max(1, Math.floor(width)), h = Math.max(0, Math.floor(height));
-  const blocks = sidebarBlocks(model, w).filter(block => !block.list || block.items.length > 0);
+  const blocks = sidebarBlocks(model, w).filter(block => !block.list || block.items.length > 0 || block.keep);
   const plan: { block: Block; rows: number }[] = [];
   let used = 0;
   for (const block of blocks) {
     const minimum = block.list ? Math.min(block.items.length, 2) : 0;
-    const need = (plan.length ? 1 : 0) + block.head.length + minimum;
+    const need = (plan.length ? 1 : 0) + block.head.length + minimum + (block.tail?.length ?? 0);
     if (used + need > h) continue;
     used += need;
     plan.push({ block, rows: minimum });
@@ -183,6 +207,7 @@ export function sidebarLines(model: StatusModel, width: number, height: number):
       out.push(...items.slice(0, entry.rows - 1));
       out.push([{ text: `+${items.length - entry.rows + 1} more`, role: "muted" }]);
     }
+    out.push(...entry.block.tail ?? []);
   });
   return out.slice(0, h).map(line => clipSegs(line, w));
 }
@@ -201,34 +226,37 @@ function sidebarBlocks(model: StatusModel, w: number): Block[] {
       ...(model.git.conflicted ? [[{ text: `${model.git.conflicted} conflicts`, role: "warning" }] as Line] : []),
       ...(model.git.ahead || model.git.behind ? [[{ text: `↑${model.git.ahead} ↓${model.git.behind}`, role: "muted" }] as Line] : [])] : model.git === null ? [[{ text: "Git unavailable", role: "muted" }] as Line] : []),
   ] });
+  const goal = model.goal;
+  if (goal) blocks.push(goalBlock(goal, w, row));
+  else if (model.checks) blocks.push({ list: false, items: [], head: [checksLine(model.checks, w)] });
   const context = model.context;
-  if (model.goal) blocks.push({ list: false, items: [], head: [
-    [{ text: "Goal", role: "text", bold: true }, { text: ` · ${model.goal.status}`, role: model.goal.status === "verified" ? "success" : "warning" }],
-    ...wrap(firstLine(model.goal.text), w, 2).map(text => [{ text, role: "text" }] as Line),
-    [{ text: model.goal.execution, role: "muted" }],
-    [{ text: `${model.goal.observations} observations · ${model.goal.retries} retries`, role: "muted" }],
-    [{ text: "/control · goal, evidence, context", role: "muted" }],
-  ] });
   if (context) {
     const share = contextShare(context), est = context.estimated ? "~" : "";
     const filled = share === undefined ? 0 : Math.max(share > 0 ? 1 : 0, Math.round(share * w));
     const tone = share !== undefined && share >= CONTEXT_WARNING ? "warning" : "accent";
     const shareText = shareSeg(context);
+    // Two budgets, each named: the segment cut (what the bar shows) and the model window. Both shares
+    // ride on the heading, both denominators on one row below the bar.
+    const modelShare = context.window && context.trigger !== null ? `${est}${Math.round(context.used / context.window * 100)}%` : undefined;
+    const segment = context.trigger !== null ? `seg ${est}${compact(context.used)} / ${est}${compact(context.trigger)}` : `${est}${compact(context.used)} · no cut`;
     blocks.push({ list: false, items: [], head: [
-      row([{ text: "Context", role: "text", bold: true }], shareText ? [{ text: context.trigger === null ? "model " : "seg ", role: "muted" }, shareText] : []),
+      row([{ text: "Context", role: "text", bold: true }], [
+        ...(shareText ? [{ text: context.trigger === null ? "model " : "seg ", role: "muted" as const }, shareText] : []),
+        ...(modelShare ? [{ text: ` · model ${modelShare}`, role: "muted" as const }] : [])]),
       ...(share === undefined ? [] : [[{ text: "━".repeat(filled), role: tone }, { text: "─".repeat(Math.max(0, w - filled)), role: "line" }] as Line]),
-      [{ text: `Segment ${est}${compact(context.used)}${context.trigger !== null ? ` / ${est}${compact(context.trigger)}` : " · cut disabled"}`, role: "text" }],
-      [{ text: context.window ? `Model ${est}${compact(context.used)} / ${compact(context.window)} · ${est}${Math.round(context.used / context.window * 100)}%` : "Model window unknown", role: "muted" }],
-      [{ text: `node ${context.segment}`, role: "muted" }],
+      row([{ text: segment, role: "muted" }], [{ text: context.window ? `model ${compact(context.window)}` : "window unknown", role: "muted" }]),
     ] });
   }
   const usage = model.usage, tools = model.tools;
   const toolCount = tools.done + tools.failed + tools.running;
   if (usage || toolCount) {
+    const figures = usage ? [`↑${compact(usage.input)} ↓${compact(usage.output)}`, `cache ${Math.round(usage.cacheRate * 100)}%`,
+      ...(usage.speed !== undefined ? [speedText(usage.speed)] : [])].join(" · ") : "";
+    // Heading with the cost; one row of figures below it. Together they need about 49 columns, more than the 34-42 the sidebar has.
     blocks.push({ list: false, items: [], head: [
       row([{ text: "Usage", role: "text", bold: true }], usage?.cost ? [{ text: usage.cost, role: "text" }] : []),
-      ...(usage ? [[{ text: `↑${compact(usage.input)} ↓${compact(usage.output)}`, role: "text" }, { text: `  cache ${Math.round(usage.cacheRate * 100)}%`, role: "muted" }] as Line] : []),
-      ...(toolCount ? [[
+      ...(figures ? [[{ text: figures, role: "muted" }] as Line] : []),
+      ...(tools.failed || tools.running ? [[
         { text: `${tools.done} done`, role: "muted" },
         ...(tools.failed ? [{ text: " · ", role: "muted" as const }, { text: `${tools.failed} failed`, role: "error" as const }] : []),
         ...(tools.running ? [{ text: " · ", role: "muted" as const }, { text: `${tools.running} running`, role: "accent" as const }] : []),
@@ -266,6 +294,60 @@ function sidebarBlocks(model: StatusModel, w: number): Block[] {
   return blocks;
 }
 
+/** `42 tok/s`; under 10 one decimal, so a slow upstream does not read as zero. */
+export function speedText(speed: number): string {
+  return `${speed < 10 ? speed.toFixed(1) : Math.round(speed)} tok/s`;
+}
+
+const PHASE: Record<GoalStatusView["phase"], { label: string; role: Seg["role"] }> = {
+  drafting: { label: "drafting", role: "accent" }, working: { label: "working", role: "accent" },
+  needs_you: { label: "needs you", role: "warning" }, met: { label: "met", role: "success" }, stopped: { label: "stopped", role: "warning" },
+};
+
+/**
+ * `Goal · needs you   4m 12s`, the objective, one row per accepted check with
+ * what it needs, the evidence ladder and what the goal has spent. A met goal
+ * is one summary row. The sidebar may cut the checks; a cut ends in "+N more".
+ */
+function goalBlock(goal: GoalStatusView, w: number, row: (left: readonly Seg[], right?: readonly Seg[]) => Line): Block {
+  const phase = PHASE[goal.phase];
+  const passed = goal.checks.filter(check => check.status === "pass" || check.status === "accepted").length;
+  const elapsed = goal.phase === "drafting" ? [] : [{ text: formatElapsed(goal.elapsedMs), role: "muted" as const }];
+  const title = [{ text: "Goal", role: "text" as const, bold: true }, { text: ` · ${phase.label}`, role: phase.role }];
+  if (goal.phase === "met") return { list: false, items: [], head: [row(title, [{ text: `${passed}/${goal.checks.length} · `, role: "muted" }, ...elapsed])] };
+  const items = goal.checks.map(check => {
+    const done = check.status === "pass" || check.status === "accepted";
+    const glyph: Seg = done ? { text: "✓ ", role: "success" } : check.status === "fail" ? { text: "✗ ", role: "error" } : { text: "○ ", role: "muted" };
+    const need = check.kind === "human" && !done ? `/accept ${check.id}` : check.status === "fail" && check.exit !== undefined ? `exit ${check.exit}`
+      : check.status === "stale" ? "rerun" : check.guard ? "guard" : "";
+    return row([glyph, { text: check.id, role: "text" }], need ? [{ text: need, role: check.kind === "human" && !done ? "warning" : "muted" }] : []);
+  });
+  const spent = [`${goal.requests} ${goal.requests === 1 ? "request" : "requests"}`, ...(goal.tokens ? [`${compact(goal.tokens)} tokens this run`] : [])].join(" · ");
+  return { list: true, keep: true, head: [row(title, elapsed), ...wrap(firstLine(goal.objective), w, 2).map(text => [{ text, role: "text" }] as Line)],
+    items, ...(goal.phase === "drafting" ? {} : { tail: [...(goal.ladder ? [[{ text: goal.ladder.glyphs, role: "text" }] as Line] : []), [{ text: spent, role: "muted" }] as Line] }) };
+}
+
+/** Without a goal: whether a check has passed since this request's last edit. */
+function checksLine(checks: NonNullable<StatusModel["checks"]>, w: number): Line {
+  if (checks.status === "unchecked" || !checks.command) return [{ text: "Checks", role: "text", bold: true }, { text: " · not run since last edit", role: "warning" }];
+  // partial: some of a declared contract holds; the rest may only be waiting for the user, so it is not a failure mark.
+  const mark: Seg[] = checks.status === "pass" ? [{ text: " ✓", role: "success" }] : checks.status === "partial" ? []
+    : checks.status === "fail" ? [{ text: " ✗", role: "error" }, ...(checks.exit === undefined ? [] : [{ text: ` exit ${checks.exit}`, role: "muted" as const }])]
+    : [{ text: " stale", role: "warning" }];
+  const room = Math.max(1, w - segsWidth(mark) - "Checks · ".length);
+  return [{ text: "Checks", role: "text", bold: true }, { text: ` · ${clipText(checks.command, room)}`, role: "muted" }, ...mark];
+}
+
+/** Without a sidebar the goal rides on the input box's bottom edge: `goal 2/3 · needs you · 4m 12s`. */
+export function goalLabel(model: StatusModel): Label | undefined {
+  const goal = model.goal;
+  if (!goal) return undefined;
+  const passed = goal.checks.filter(check => check.status === "pass" || check.status === "accepted").length;
+  const parts = [`goal ${goal.phase === "drafting" ? "" : `${passed}/${goal.checks.length} · `}${PHASE[goal.phase].label}`,
+    ...(goal.phase === "drafting" ? [] : [formatElapsed(goal.elapsedMs)])];
+  return { segs: [{ text: parts.join(" · "), role: goal.phase === "needs_you" ? "warning" : "muted" }], priority: 2, min: 8 };
+}
+
 function fileLine(file: FileStatus, w: number): Line {
   const name = file.path.split("/").pop() || file.path;
   const counts: Seg[] = file.edited && (file.added || file.removed)

package/src/core/benchmark.ts

@@ -1,195 +0,0 @@
-/** Evaluation contracts, separate from production policies (research M.3). */
-import { shuffled } from "./experiment.ts";
-
-export const LONGBENCH_SOURCE = {
-  revision: "2b48e494f2c7a2f0af81aae178e05c7e1dde0fe9",
-  sha256: "15d61c22d92c96900b3c4948b6aeea218d3214b676a65df48e7b8555604c7fe2",
-  scorerRevision: "2e00731f8d0bff23dc4325161044d0ed8af94c1e",
-} as const;
-
-export const BENCHMARK_ARMS = ["node", "replay", "mask"] as const;
-export type BenchmarkArm = typeof BENCHMARK_ARMS[number];
-export type Choice = "A" | "B" | "C" | "D";
-export interface BenchmarkItem {
-  _id: string;
-  domain: string;
-  sub_domain: string;
-  difficulty: "easy" | "hard";
-  length: "short" | "medium" | "long";
-  question: string;
-  choice_A: string;
-  choice_B: string;
-  choice_C: string;
-  choice_D: string;
-  answer: Choice;
-  context: string;
-}
-
-function isBenchmarkItem(value: unknown): value is BenchmarkItem {
-  if (typeof value !== "object" || value === null) return false;
-  const row = value as Record<string, unknown>;
-  for (const key of ["_id", "domain", "sub_domain", "question", "choice_A", "choice_B", "choice_C", "choice_D", "context"]) {
-    if (typeof row[key] !== "string" || row[key].length === 0) return false;
-  }
-  return (row["answer"] === "A" || row["answer"] === "B" || row["answer"] === "C" || row["answer"] === "D") &&
-    (row["difficulty"] === "easy" || row["difficulty"] === "hard") &&
-    (row["length"] === "short" || row["length"] === "medium" || row["length"] === "long");
-}
-
-export function parseBenchmark(data: unknown): BenchmarkItem[] {
-  if (!Array.isArray(data) || data.length === 0) throw new Error("empty or invalid benchmark");
-  const ids = new Set<string>();
-  return data.map((item: unknown) => {
-    if (!isBenchmarkItem(item)) throw new Error("invalid benchmark item");
-    if (ids.has(item._id)) throw new Error(`duplicate benchmark id: ${item._id}`);
-    ids.add(item._id);
-    return item;
-  });
-}
-
-/** Exact LongBench v2 pred.py extraction, pinned in docs/benchmarks.md (MIT). */
-export function extractBenchmarkAnswer(response: string): Choice | null {
-  const plain = response.replaceAll("*", "");
-  return (plain.match(/The correct answer is \(([A-D])\)/)?.[1] ??
-    plain.match(/The correct answer is ([A-D])/)?.[1] ?? null) as Choice | null;
-}
-
-/** Domain-balanced sampling; neither correctness nor answer labels enter selection. */
-export function sampleBenchmark(items: readonly BenchmarkItem[], count: number, seed: number): BenchmarkItem[] {
-  if (!Number.isSafeInteger(count) || count < 1 || count > items.length) throw new RangeError("invalid sample count");
-  const domains = [...new Set(items.map((item) => item.domain))].sort();
-  const buckets = shuffled(domains, seed).map((domain, index) => shuffled(
-    items.filter((item) => item.domain === domain).toSorted((a, b) => a._id.localeCompare(b._id)), seed + index + 1,
-  ));
-  const out: BenchmarkItem[] = [];
-  for (let round = 0; out.length < count; round++) {
-    for (const bucket of buckets) {
-      const item = bucket[round];
-      if (item !== undefined && out.length < count) out.push(item);
-    }
-  }
-  return out;
-}
-
-export interface BenchmarkJob { id: string; arm: BenchmarkArm }
-export type HarnessProfile = "control" | "enhanced";
-export interface HarnessJob { id: string; arm: "node" | "replay"; profile: HarnessProfile }
-
-/** A Latin rotation balances each treatment's position before running any model. */
-export function harnessSchedule(items: readonly BenchmarkItem[], seed: number): HarnessJob[] {
-  const treatments = shuffled([
-    { arm: "node", profile: "control" }, { arm: "replay", profile: "control" },
-    { arm: "node", profile: "enhanced" }, { arm: "replay", profile: "enhanced" },
-  ] as const, seed);
-  const offsets = shuffled(items.map((_, index) => index % treatments.length), seed + 1);
-  return items.flatMap((item, index) => treatments.map((_, position) => ({
-    id: item._id, ...treatments[(position + offsets[index]!) % treatments.length]!,
-  })));
-}
-
-export function summarizeHarness(plan: readonly HarnessJob[], results: readonly (BenchmarkOutcome & { profile: HarnessProfile })[]) {
-  return (["control", "enhanced"] as const).flatMap((profile) => summarizeBenchmark(
-    plan.filter((job) => job.profile === profile), results.filter((result) => result.profile === profile),
-  ).filter((row) => row.planned > 0).map((row) => ({ profile, ...row })));
-}
-/** Pair nearby in time, randomize treatment order inside each task, once before running. */
-export function benchmarkSchedule(items: readonly BenchmarkItem[], seed: number): BenchmarkJob[] {
-  // Balance all six orders before shuffling. Adjacent LCG seeds yield correlated
-  // first draws and can accidentally put the same arm last in every block.
-  const orders: readonly (readonly BenchmarkArm[])[] = [
-    ["node", "replay", "mask"], ["node", "mask", "replay"], ["replay", "node", "mask"],
-    ["replay", "mask", "node"], ["mask", "node", "replay"], ["mask", "replay", "node"],
-  ];
-  const balanced = shuffled(items.map((_, index) => orders[index % orders.length]!), seed);
-  return items.flatMap((item, index) => balanced[index]!.map((arm) => ({ id: item._id, arm })));
-}
-
-/** Split losslessly, including surrogate pairs; chunks are files, not a context truncation. */
-export function benchmarkChunks(context: string, size = 16_000): string[] {
-  if (!Number.isSafeInteger(size) || size < 2) throw new RangeError("chunk size must be at least two");
-  const chunks: string[] = [];
-  for (let start = 0; start < context.length;) {
-    let end = Math.min(context.length, start + size);
-    const last = context.charCodeAt(end - 1);
-    if (end < context.length && last >= 0xd800 && last <= 0xdbff) end--;
-    chunks.push(context.slice(start, end));
-    start = end;
-  }
-  return chunks;
-}
-
-/** Explicit field selection prevents evaluator-only metadata from leaking to the agent. */
-export function benchmarkPrompt(item: BenchmarkItem, chunks: number): string {
-  return `Answer the question using the complete source document in corpus/part-00000.txt through corpus/part-${String(chunks - 1).padStart(5, "0")}.txt. ` +
-    "The numbered files concatenate in order to the original document; none of it was removed. Use read and grep to examine the evidence.\n\n" +
-    `Question: ${item.question}\nChoices:\n(A) ${item.choice_A}\n(B) ${item.choice_B}\n(C) ${item.choice_C}\n(D) ${item.choice_D}\n\n` +
-    'Format your final response as follows: "The correct answer is (insert answer here)".';
-}
-
-export interface BenchmarkPrice { input: number; output: number; cacheRead: number; cacheWrite?: number }
-export interface BenchmarkUsage { input_tokens: number; output_tokens: number; cache_read_input_tokens: number; cache_creation_input_tokens: number }
-
-/** A conservative allowance, not a tokenizer: wire bytes plus framing, at uncached peak rates. */
-export function requestReservation(wireBytes: number, maxOutput: number, price: BenchmarkPrice): number {
-  if (!Number.isSafeInteger(wireBytes) || wireBytes < 0 || !Number.isSafeInteger(maxOutput) || maxOutput < 1) throw new RangeError("invalid request size");
-  return ((wireBytes + 4096) * price.input + maxOutput * price.output) / 1_000_000;
-}
-
-export function measuredCharge(usage: BenchmarkUsage, price: BenchmarkPrice): number | null {
-  const counts = [usage.input_tokens, usage.output_tokens, usage.cache_read_input_tokens, usage.cache_creation_input_tokens];
-  if (counts.some((value) => !Number.isSafeInteger(value) || value < 0) ||
-      usage.input_tokens + usage.cache_read_input_tokens + usage.cache_creation_input_tokens === 0 || usage.output_tokens === 0) return null;
-  return (usage.input_tokens * price.input + usage.cache_creation_input_tokens * (price.cacheWrite ?? price.input) +
-    usage.cache_read_input_tokens * price.cacheRead + usage.output_tokens * price.output) / 1_000_000;
-}
-
-export function canReserve(charged: number, reserve: number, budget: number): boolean {
-  if (![charged, reserve, budget].every(Number.isFinite) || charged < 0 || reserve <= 0 || budget <= 0) return false;
-  return charged + reserve <= budget;
-}
-
-/** Unknown outcomes keep their reservation; an unexpectedly high bill is never clipped. */
-export function settledCharge(reserve: number, measured: number | null): number {
-  return measured === null ? reserve : measured;
-}
-
-export function wilsonInterval(correct: number, total: number): [number, number] | null {
-  if (!Number.isSafeInteger(total) || !Number.isSafeInteger(correct) || correct < 0 || total < correct) throw new RangeError("invalid score");
-  if (total === 0) return null;
-  const z = 1.959963984540054;
-  const p = correct / total;
-  const d = 1 + z * z / total;
-  const center = (p + z * z / (2 * total)) / d;
-  const radius = z * Math.sqrt(p * (1 - p) / total + z * z / (4 * total * total)) / d;
-  return [Math.max(0, center - radius), Math.min(1, center + radius)];
-}
-
-export interface BenchmarkOutcome {
-  id: string;
-  arm: BenchmarkArm;
-  status: "answered" | "error" | "budget" | "stopped";
-  correct: boolean;
-}
-
-export function summarizeBenchmark(plan: readonly BenchmarkJob[], outcomes: readonly BenchmarkOutcome[]) {
-  const planned = new Set(plan.map((job) => `${job.id}/${job.arm}`));
-  const seen = new Set<string>();
-  for (const outcome of outcomes) {
-    const key = `${outcome.id}/${outcome.arm}`;
-    if (!planned.has(key) || seen.has(key)) throw new Error("unknown or duplicate benchmark outcome");
-    if (outcome.correct && outcome.status !== "answered") throw new Error("unfinished result cannot be correct");
-    seen.add(key);
-  }
-  return BENCHMARK_ARMS.map((arm) => {
-    const rows = outcomes.filter((row) => row.arm === arm);
-    const total = plan.filter((job) => job.arm === arm).length;
-    const correct = rows.filter((row) => row.correct).length;
-    const attempted = rows.length;
-    const answered = rows.filter((row) => row.status === "answered").length;
-    return { arm, planned: total, attempted, answered, correct, missing: total - attempted,
-      accuracy: attempted > 0 ? correct / attempted : null,
-      interval95: wilsonInterval(correct, attempted),
-      plannedSolvedFraction: total > 0 ? correct / total : null,
-      complete: attempted === total };
-  });
-}

package/src/core/evaluation-coverage.ts

@@ -1,74 +0,0 @@
-/** R12c: missing final rows cannot erase attempted work or measured charges. */
-import type { EvalJob, EvalResult } from "./experiment.ts";
-import { recoveryCase } from "./recovery-corpus.ts";
-import { TOOL_MODE_FACTORS, isToolModeTask, isToolModeMetrics } from "./tool-mode-protocol.ts";
-
-export interface BillingRecord {
-  kind: "reserved" | "settled"; request: number; inputSha256: string;
-  reservation: number; charge: number; measured: number | null; ledgerCharged: number;
-}
-export function requestAccounting(records: readonly BillingRecord[], before: number) {
-  const nonnegative = (n: unknown): n is number => typeof n === "number" && Number.isFinite(n) && n >= 0;
-  if (!nonnegative(before)) throw Error("invalid trial ledger start");
-  let charged = before, pending: BillingRecord | undefined, requests = 0, known = 0, unknown = 0;
-  for (const row of records) {
-    if (![row.reservation, row.charge, row.ledgerCharged].every(nonnegative) || row.measured !== null && !nonnegative(row.measured) ||
-        !Number.isSafeInteger(row.request) || row.request < 1 || !/^[a-f0-9]{64}$/.test(row.inputSha256)) throw Error("invalid billing record");
-    if (row.kind === "reserved") {
-      if (pending || row.request !== ++requests || row.measured !== null || row.charge !== row.reservation) throw Error("invalid reservation order");
-      pending = row; charged += row.reservation;
-    } else if (row.kind === "settled") {
-      if (!pending || row.request !== pending.request || row.inputSha256 !== pending.inputSha256 || row.reservation !== pending.reservation ||
-          row.charge !== (row.measured ?? row.reservation)) throw Error("invalid settlement identity");
-      charged += row.charge - pending.reservation;
-      if (row.measured === null) unknown += row.reservation; else known += row.measured;
-      pending = undefined;
-    } else throw Error("invalid billing record kind");
-    if (row.ledgerCharged !== charged) throw Error("billing ledger sequence changed");
-  }
-  return { requests, charged, knownCostSubtotal: known, unsettledRequests: pending ? 1 : 0,
-    unsettledReservation: pending?.reservation ?? 0, unknownSettlementReserve: unknown,
-    fullyMeasured: !pending && records.filter(row => row.kind === "settled").every(row => row.measured !== null) };
-}
-export type RequestAccounting = ReturnType<typeof requestAccounting>;
-export type TrialCoverage = { id: string; accounting: RequestAccounting; tail: "complete" | "truncated";
-  result?: EvalResult; interruption: "deadline" | "caller" | null; partialMetrics?: EvalResult["recovery"] };
-
-export function summarizeRecoveryCoverage(jobs: readonly EvalJob[], trials: readonly TrialCoverage[], mechanical: boolean) {
-  const ids = new Set(jobs.map(job => job.id)), byId = new Map(trials.map(trial => [trial.id, trial]));
-  if (ids.size !== jobs.length || byId.size !== trials.length || trials.some(trial => !ids.has(trial.id) || trial.result && trial.result.id !== trial.id)) throw Error("invalid recovery coverage identities");
-  const group = (selected: readonly EvalJob[]) => {
-    const rows = selected.flatMap(job => byId.has(job.id) ? [byId.get(job.id)!] : []), finalized = rows.filter(row => row.result);
-    const passed = finalized.filter(row => row.result!.scenarioPassed).length;
-    const unknown = rows.filter(row => !row.result || row.tail !== "complete" || !row.accounting.fullyMeasured).length;
-    const known = rows.reduce((n, row) => n + row.accounting.knownCostSubtotal, 0);
-    const cost = !mechanical && rows.length === selected.length && unknown === 0 ? known : null;
-    const metrics = rows.flatMap(row => row.result?.recovery ?? row.partialMetrics ?? []);
-    const modes = finalized.flatMap(row => row.result?.toolMode ?? []);
-    if (modes.some(mode => !isToolModeMetrics(mode))) throw Error("invalid tool mode measurements");
-    const toolMode = selected.some(job => isToolModeTask(job.task)) ? { observed: modes.length, missing: selected.length - modes.length,
-      queryCalls: modes.reduce((n, mode) => n + mode.queryCalls, 0),
-      primitiveCalls: modes.reduce((n, mode) => n + mode.primitiveCalls, 0), sourcesComplete: modes.filter(mode => mode.sourceComplete).length,
-      meanElapsedMs: modes.length === selected.length && modes.length > 0 ? modes.reduce((n, mode) => n + mode.elapsedMs, 0) / modes.length : null } : undefined;
-    return { ...(toolMode ? { toolMode } : {}), planned: selected.length, attempted: rows.length, finalized: finalized.length, incomplete: rows.length - finalized.length,
-      unattempted: selected.length - rows.length, failed: finalized.length - passed, passed, plannedPassFraction: selected.length ? passed / selected.length : null,
-      cancelled: rows.filter(row => row.interruption === "caller").length, timedOut: rows.filter(row => row.interruption === "deadline").length,
-      requests: rows.reduce((n, row) => n + row.accounting.requests, 0), truncatedRequestLogs: rows.filter(row => row.tail === "truncated").length,
-      knownCostSubtotal: mechanical ? null : known, costUsd: cost,
-      unsettledRequests: rows.reduce((n, row) => n + row.accounting.unsettledRequests, 0),
-      unsettledReservationUsd: mechanical ? null : rows.reduce((n, row) => n + row.accounting.unsettledReservation, 0),
-      unknownSettlementReserveUsd: mechanical ? null : rows.reduce((n, row) => n + row.accounting.unknownSettlementReserve, 0),
-      unknownCostTrials: mechanical ? rows.length : unknown, costPerScenarioSuccess: cost !== null && passed > 0 ? cost / passed : null,
-      unverifiedEndings: finalized.filter(row => row.result!.unverifiedEnding).length,
-      recovery: { observed: metrics.length, missing: selected.length - metrics.length,
-        firstToolSuccess: metrics.filter(row => row.firstToolSuccess === true).length,
-        firstToolUnknown: metrics.filter(row => row.firstToolSuccess === null).length,
-        firstPassTaskSuccess: metrics.filter(row => row.firstPassTaskSuccess === true).length,
-        ...Object.fromEntries((["toolCalls", "outerCalls", "nestedCalls", "unresolvedCalls", "toolFailures", "classifiedFailures", "unclassifiedFailures", "recoveryCalls", "unchangedFailureRetries", "originalEvidenceRetrievals", "unsupportedClaims"] as const)
-          .map(key => [key, metrics.reduce((n, row) => n + row[key], 0)])) } };
-  };
-  const toolModes = jobs.some(job => isToolModeTask(job.task));
-  return { ...group(jobs), arms: (toolModes ? TOOL_MODE_FACTORS : ["profile", "workingSet", "strictVerification"]).flatMap(factor => ["control", "treatment"].map(arm =>
-    ({ factor, arm, ...group(jobs.filter(job => job.factor === factor && job.arm === arm)) }))),
-    partitions: toolModes ? [] : ["development", "holdout"].map(partition => ({ partition, ...group(jobs.filter(job => recoveryCase(job.task)?.partition === partition)) })) };
-}

package/src/core/experiment.ts

@@ -1,173 +0,0 @@
-/** Experimental projections never mutate the durable event stream (D8, L). */
-import type { Message } from "./project.ts";
-import { isTrialLifecycle, type TrialLifecycle } from "./trial-lifecycle.ts";
-import { RECOVERY_TASKS, recoveryCase, type RecoveryTask } from "./recovery-corpus.ts";
-import { isRecoveryMetrics, type RecoveryMetrics } from "./recovery-evaluation.ts";
-import { TOOL_MODE_FACTORS, TOOL_MODE_TASKS, isToolModeTask, isToolModeMetrics, type ToolModeTask, type ToolModeMetrics } from "./tool-mode-protocol.ts";
-
-export type ContextArm = "node" | "replay" | "truncate" | "mask";
-
-export function experimentalContext(messages: readonly Message[], arm: ContextArm, keepRounds: number): Message[] {
-  if (!Number.isSafeInteger(keepRounds) || keepRounds < 1) throw new RangeError("keepRounds must be positive");
-  if (arm === "node" || arm === "replay") return [...messages];
-  const groups: Message[][] = [[]];
-  for (const message of messages) {
-    if (message.role === "assistant") groups.push([]);
-    groups.at(-1)!.push(message);
-  }
-  const recent = Math.max(1, groups.length - keepRounds);
-  if (arm === "truncate") return [...groups[0]!, ...groups.slice(recent).flat()];
-  return groups.flatMap((group, i) => i >= recent ? group : group.map((message) => ({
-    ...message,
-    content: message.content.map((block) => block.type === "tool_result"
-      ? { ...block, content: "[observation masked by experiment; original remains in log]" } : block),
-  })));
-}
-
-/** Fisher-Yates with a fixed seed; scheduling and intervals are reproducible. */
-export function seededRandom(seed: number): () => number {
-  let state = seed >>> 0;
-  return () => {
-    state = (Math.imul(1664525, state) + 1013904223) >>> 0;
-    return state / 4294967296;
-  };
-}
-
-export function shuffled<T>(items: readonly T[], seed: number): T[] {
-  const random = seededRandom(seed);
-  const out = [...items];
-  for (let i = out.length - 1; i > 0; i--) {
-    const j = Math.floor(random() * (i + 1));
-    [out[i], out[j]] = [out[j]!, out[i]!];
-  }
-  return out;
-}
-
-/** O.16: each contrast changes exactly one policy, with paired task repeats. */
-export const EVAL_FACTORS = ["profile", "workingSet", "strictVerification"] as const;
-export const EVAL_TASKS = ["short", "boundary-recall", "false-completion", "human-boundary"] as const;
-const supportedFactors = [...EVAL_FACTORS, ...TOOL_MODE_FACTORS] as const;
-export type EvalFactor = typeof supportedFactors[number];
-export type EvalTask = typeof EVAL_TASKS[number] | RecoveryTask | ToolModeTask;
-const supportedTasks:readonly EvalTask[]=[...EVAL_TASKS,...RECOVERY_TASKS,...TOOL_MODE_TASKS];
-export type EvalArm = "control" | "treatment";
-export type EvalPolicy = { profile: "full" | "coding"; workingSet: boolean; strictVerification: boolean; querySyntax?: "json" | "sql" };
-export type EvalJob = { id: string; factor: EvalFactor; task: EvalTask; repeat: number; arm: EvalArm; policy: EvalPolicy };
-export function evalPolicy(factor: EvalFactor, arm: EvalArm): EvalPolicy {
-  const policy: EvalPolicy = { profile: "coding", workingSet: true, strictVerification: true };
-  if (factor === "querySyntax") return { ...policy, querySyntax: arm === "treatment" ? "sql" : "json" };
-  if (arm === "control") {
-    if (factor === "profile") policy.profile = "full";
-    else policy[factor] = false;
-  }
-  return policy;
-}
-export function singleFactorSchedule(seed: number, repeats = 1, tasks: readonly EvalTask[] = EVAL_TASKS, factors: readonly EvalFactor[] = EVAL_FACTORS): EvalJob[] {
-  if (!Number.isSafeInteger(seed) || !Number.isSafeInteger(repeats) || repeats < 1 || repeats > 20 || !tasks.length ||
-      new Set(tasks).size !== tasks.length || tasks.some(task => !supportedTasks.includes(task)) || !factors.length || new Set(factors).size !== factors.length ||
-      factors.some(factor => !supportedFactors.includes(factor) || tasks.some(task => isToolModeTask(task) !== (TOOL_MODE_FACTORS as readonly string[]).includes(factor)))) throw new RangeError("invalid eval schedule");
-  const pairs = shuffled(factors.flatMap(factor => [...tasks].sort().flatMap(task =>
-    Array.from({ length: repeats }, (_, repeat) => ({ factor, task, repeat })))), seed);
-  // Alternate first position before shuffling: adjacent LCG seeds are correlated.
-  const positions = new Map<EvalFactor, number>();
-  const order = shuffled(["control", "treatment"] as const, seed + 1);
-  return pairs.flatMap(pair => {
-    const offset = positions.get(pair.factor) ?? 0; positions.set(pair.factor, offset + 1);
-    return order.map((_, i) => {
-      const arm = order[(offset + i) % 2]!;
-      return { ...pair, arm, id: `${pair.factor}/${pair.task}/${pair.repeat}/${arm}`, policy: evalPolicy(pair.factor, arm) };
-    });
-  });
-}
-export function validateEvalJobs(jobs: readonly EvalJob[]): void {
-  const seen = new Set<string>();
-  for (const job of jobs) {
-    if (!supportedFactors.includes(job.factor) || !supportedTasks.includes(job.task) || !["control", "treatment"].includes(job.arm) ||
-        isToolModeTask(job.task) !== (TOOL_MODE_FACTORS as readonly string[]).includes(job.factor) ||
-        !Number.isSafeInteger(job.repeat) || job.repeat < 0 || job.id !== `${job.factor}/${job.task}/${job.repeat}/${job.arm}` || seen.has(job.id) ||
-        JSON.stringify(job.policy) !== JSON.stringify(evalPolicy(job.factor, job.arm))) throw new Error("invalid or confounded eval job");
-    seen.add(job.id);
-  }
-  for (const job of jobs) if (!seen.has(`${job.factor}/${job.task}/${job.repeat}/${job.arm === "control" ? "treatment" : "control"}`)) throw new Error("unpaired eval job");
-}
-export interface EvalResult {
-  toolMode?: ToolModeMetrics;
-  recovery?: RecoveryMetrics;
-  /** v2 separates an unmet final state from a machine-readable unsupported claim. */
-  unverifiedEnding?: boolean;
-  /** Absent in the frozen O.16 archives; absence is unknown, not an uninterrupted run. */
-  lifecycle?: TrialLifecycle;
-  id: string;
-  status: "finished" | "error" | "budget" | "stopped";
-  scenarioPassed: boolean;
-  artifactCorrect: boolean;
-  verified: boolean;
-  awaitingHuman: boolean;
-  falseCompletion: boolean;
-  requests: number;
-  toolFailures: number;
-  repeatedReads: number;
-  promptTokens: number | null;
-  cacheTokens: number | null;
-  outputTokens: number | null;
-  costUsd: number | null;
-  reservedUnknownUsd: number;
-}
-export function summarizeSingleFactor(jobs: readonly EvalJob[], results: readonly EvalResult[]) {
-  validateEvalJobs(jobs);
-  const byId = new Map<string, EvalResult>(), planned = new Set(jobs.map(job => job.id));
-  for (const row of results) {
-    if (row.toolMode !== undefined && (!isToolModeMetrics(row.toolMode) || row.scenarioPassed && !row.toolMode.sourceComplete)) throw Error("invalid tool mode metrics");
-    if (row.unverifiedEnding !== undefined && typeof row.unverifiedEnding !== "boolean") throw Error("invalid ending metric");
-    if(row.recovery!==undefined&&(!isRecoveryMetrics(row.recovery)||row.recovery.finalTaskSuccess!==row.scenarioPassed||row.recovery.toolFailures!==row.toolFailures||row.falseCompletion!==(row.recovery.unsupportedClaims>0)))throw Error("invalid recovery metrics");
-    if (row.lifecycle !== undefined && (!isTrialLifecycle(row.lifecycle) || row.lifecycle.backendRequests > row.requests ||
-        row.lifecycle.interruption !== null && row.status === "finished" || row.lifecycle.cleanup === "failed" && row.status !== "error"))
-      throw new Error("invalid eval lifecycle result");
-    if (!planned.has(row.id) || byId.has(row.id) || !["finished", "error", "budget", "stopped"].includes(row.status) ||
-        [row.scenarioPassed, row.artifactCorrect, row.verified, row.awaitingHuman, row.falseCompletion].some(value => typeof value !== "boolean") ||
-        row.scenarioPassed && (row.status !== "finished" || !row.verified || row.falseCompletion && row.recovery === undefined) ||
-        [row.requests, row.toolFailures, row.repeatedReads].some(n => !Number.isSafeInteger(n) || n < 0) ||
-        [row.promptTokens, row.cacheTokens, row.outputTokens].some(n => n !== null && (!Number.isSafeInteger(n) || n < 0)) ||
-        [row.costUsd, row.reservedUnknownUsd].some(n => n !== null && (!Number.isFinite(n) || n < 0)) ||
-        row.cacheTokens !== null && row.promptTokens !== null && row.cacheTokens > row.promptTokens) throw new Error("invalid or duplicate eval result");
-    byId.set(row.id, row);
-  }
-  const factors = supportedFactors.filter(factor => jobs.some(job => job.factor === factor));
-  const arms = factors.flatMap(factor => (["control", "treatment"] as const).map(arm => {
-    const selected = jobs.filter(job => job.factor === factor && job.arm === arm), rows = selected.flatMap(job => byId.has(job.id) ? [byId.get(job.id)!] : []);
-    const complete = rows.length === selected.length, scenarioPassed = rows.filter(row => row.scenarioPassed).length;
-    const knownCostSubtotal = rows.reduce((sum, row) => sum + (row.costUsd ?? 0), 0);
-    // Missing jobs and unknown charges cannot become a zero-priced success.
-    const costUsd = complete && rows.length > 0 && rows.every(row => row.costUsd !== null) ? knownCostSubtotal : null;
-    const total = (field: "promptTokens" | "cacheTokens" | "outputTokens") => complete && rows.length > 0 && rows.every(row => row[field] !== null)
-      ? rows.reduce((sum, row) => sum + row[field]!, 0) : null;
-    const recovery=selected.some(job=>recoveryCase(job.task)||isToolModeTask(job.task))?{observed:rows.filter(row=>row.recovery).length,
-      missing:selected.length-rows.filter(row=>row.recovery).length,
-      firstToolSuccess:rows.filter(row=>row.recovery?.firstToolSuccess===true).length,
-      firstPassTaskSuccess:rows.filter(row=>row.recovery?.firstPassTaskSuccess===true).length,
-      ...Object.fromEntries((["toolCalls","outerCalls","nestedCalls","unresolvedCalls","recoveryCalls","unchangedFailureRetries","originalEvidenceRetrievals","unsupportedClaims"] as const)
-        .map(key=>[key,rows.reduce((n,row)=>n+(row.recovery?.[key]??0),0)]))}:undefined;
-    return { factor, arm, ...(recovery?{recovery}:{}), planned: selected.length, attempted: rows.length, missing: selected.length - rows.length,
-      failed: rows.filter(row => !row.scenarioPassed).length, scenarioPassed, verified: rows.filter(row => row.verified).length,
-      awaitingHuman: rows.filter(row => row.awaitingHuman).length, falseCompletions: rows.filter(row => row.falseCompletion).length,
-      plannedPassFraction: selected.length ? scenarioPassed / selected.length : null, complete,
-      requests: rows.reduce((n, row) => n + row.requests, 0), toolFailures: rows.reduce((n, row) => n + row.toolFailures, 0),
-      repeatedReads: rows.reduce((n, row) => n + row.repeatedReads, 0), promptTokens: total("promptTokens"), cacheTokens: total("cacheTokens"), outputTokens: total("outputTokens"),
-      knownCostSubtotal, costUsd, unknownCostJobs: rows.filter(row => row.costUsd === null).length,
-      reservedUnknownUsd: rows.reduce((n, row) => n + row.reservedUnknownUsd, 0),
-      costPerScenarioSuccess: costUsd !== null && scenarioPassed > 0 ? costUsd / scenarioPassed : null };
-  }));
-  const comparisons = factors.map(factor => {
-    const pairs = jobs.filter(job => job.factor === factor && job.arm === "control");
-    const clusters = [...new Set(pairs.map(job => job.task))].sort().map(task => {
-      const matched = pairs.filter(job => job.task === task).flatMap(job => {
-        const control = byId.get(job.id), treatment = byId.get(`${factor}/${task}/${job.repeat}/treatment`);
-        return control && treatment ? [{ delta: Number(treatment.scenarioPassed) - Number(control.scenarioPassed) }] : [];
-      });
-      return { task, matchedPairs: matched.length, passDelta: matched.length ? matched.reduce((n, pair) => n + pair.delta, 0) / matched.length : null };
-    });
-    const matchedPairs = clusters.reduce((n, row) => n + row.matchedPairs, 0);
-    return { factor, matchedPairs, missingPairs: pairs.length - matchedPairs, clusters };
-  });
-  return { arms, comparisons };
-}