@visual-z/xeno 0.7.2 → 0.8.1

package/src/runtime/native-eval.ts

@@ -1,146 +0,0 @@
-/** Q.20: meter the production Responses adapter without translating its wire protocol. */
-import { createHash } from "node:crypto";
-import { nativeMeasured, nativeReservation, nativeUsage } from "../core/native-eval.ts";
-import type { BenchmarkPrice } from "../core/benchmark.ts";
-import type { FrozenPrefix, Message } from "../core/project.ts";
-import type { Provider, ProviderConfig } from "./provider.ts";
-import { PiProvider } from "./pi-provider.ts";
-import type { ComparisonLedger, GatewayRecord } from "./harness-comparison.ts";
-
-const digest = (body: string) => createHash("sha256").update(body).digest("hex");
-const bytes = (body: string) => new TextEncoder().encode(body);
-const noEffects = { env: () => undefined, run: () => { throw Error("evaluation forbids credential commands"); } };
-// Offline replay only. It has the SDK's required claim shape, no signature or authority.
-export const NATIVE_EVAL_TEST_KEY = "offline." + btoa(JSON.stringify({ "https://api.openai.com/auth": { chatgpt_account_id: "offline-evaluation" } })) + ".unsigned";
-
-export async function nativeWire(request: Request): Promise<Record<string, unknown>> {
-  const raw = new Uint8Array(await request.clone().arrayBuffer());
-  if (raw.length > 8 * 1024 * 1024) throw Error("native eval wire exceeds archive bound");
-  const body = request.headers.get("content-encoding") === "zstd" ? Bun.zstdDecompressSync(raw) : raw;
-  if (body.length > 8 * 1024 * 1024) throw Error("native eval wire exceeds archive bound");
-  return JSON.parse(new TextDecoder().decode(body)) as Record<string, unknown>;
-}
-/** Running the same serializer with a local fetch prevents a second, approximate wire builder. */
-export async function nativePayload(config: ProviderConfig, frozen: FrozenPrefix, messages: Message[], sessionId?: string): Promise<Record<string, unknown>> {
-  let payload: Record<string, unknown> | undefined;
-  const capture = (async (input: RequestInfo | URL, init?: RequestInit) => {
-    payload = await nativeWire(new Request(input, init));
-    return new Response(null, { status: 400 });
-  }) as typeof fetch;
-  const provider = new PiProvider(config, NATIVE_EVAL_TEST_KEY, { oauth: true, effects: noEffects, fetch: capture,
-    ...(sessionId ? { sessionId } : {}), retry: { maxRetries: 0, maxRetryDelayMs: 0 } });
-  await provider.send(frozen, messages, 0).catch(() => {});
-  if (!payload) throw Error("native offline wire was not produced");
-  return payload;
-}
-export function nativeInput(payload: Record<string, unknown>): { bytes: Uint8Array; messages: unknown[] } {
-  if (!Array.isArray(payload["input"])) throw Error("native input is missing");
-  return { bytes: bytes(JSON.stringify([payload["instructions"], payload["tools"] ?? [], payload["input"]])), messages: payload["input"] };
-}
-export interface NativeResponse { model: unknown; usage: unknown; status: unknown; service_tier: unknown }
-
-/** Forward unchanged chunks with backpressure; retain only a terminal receipt, never headers. */
-export function nativeResponseTap(body: ReadableStream<Uint8Array>, terminal: (response: NativeResponse) => Promise<void>) {
-  const decoder = new TextDecoder(); let pending = "", observed = false;
-  const line = async (text: string) => {
-    if (!text.startsWith("data:") || text.slice(5).trim() === "[DONE]") return;
-    const event = JSON.parse(text.slice(5)) as Record<string, unknown>;
-    if (!["response.completed", "response.done", "response.incomplete"].includes(String(event["type"]))) return;
-    if (observed) throw Error("duplicate native terminal response");
-    observed = true;
-    const response = event["response"] as Record<string, unknown> | undefined;
-    await terminal({ model: response?.["model"] ?? null, usage: response?.["usage"] ?? null,
-      status: response?.["status"] ?? null, service_tier: response?.["service_tier"] ?? null });
-  };
-  return body.pipeThrough(new TransformStream<Uint8Array, Uint8Array>({
-    async transform(chunk, controller) {
-      pending += decoder.decode(chunk, { stream: true });
-      // The completed event repeats output. Bound this framing buffer separately from the saved receipt.
-      if (pending.length > 8 * 1024 * 1024) throw Error("native SSE frame exceeds archive bound");
-      let at: number;
-      while ((at = pending.indexOf("\n")) >= 0) { await line(pending.slice(0, at).replace(/\r$/, "")); pending = pending.slice(at + 1); }
-      controller.enqueue(chunk);
-    },
-    async flush() { pending += decoder.decode(); if (pending.trim()) await line(pending.trim()); },
-  }));
-}
-
-export function nativeEvalTransport(options: {
-  config: ProviderConfig; key: string; price: BenchmarkPrice; ledger: ComparisonLedger; maxRequests: number;
-  fetch?: typeof fetch;
-  sessionId?: string;
-  record: (row: GatewayRecord) => Promise<void>;
-  wire: (payload: Record<string, unknown>, request: number) => Promise<void>;
-  response: (receipt: NativeResponse, request: number) => Promise<void>;
-}) {
-  const { config, ledger, price } = options;
-  if (config.spec.api !== "openai-codex-responses" || !config.spec.maxTokens || config.maxTokens !== config.spec.maxTokens)
-    throw Error("native eval must reserve the full subscription model output ceiling");
-  const controller = new AbortController();
-  let requests = 0, stopped: "budget" | "balance" | "error" | "limit" | null = null;
-  let previous: ReturnType<typeof nativeInput> | null = null;
-  let current: { row: GatewayRecord; started: number; terminal: NativeResponse | null; httpStatus: number | null } | undefined;
-  let active = false, idle = Promise.resolve();
-  const observedFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
-    if (!active || current || controller.signal.aborted || ledger.halted || stopped) throw Error("native eval refused an extra request");
-    if (requests >= options.maxRequests) { stopped = "limit"; throw Error("native request limit"); }
-    const request = new Request(input, init), payload = await nativeWire(request), wire = JSON.stringify(payload);
-    if (payload["model"] !== (config.spec.wireId ?? config.spec.id) || payload["store"] !== false || payload["previous_response_id"] !== undefined || payload["stream"] !== true)
-      throw Error("native wire differs from frozen stateless policy");
-    const reservation = nativeReservation(bytes(wire).length, config.spec.maxTokens!, price);
-    if (ledger.budget === null || ledger.charged + reservation > ledger.budget) { stopped = "budget"; throw Error("native eval budget exhausted"); }
-    const inputState = nativeInput(payload); let shared = 0;
-    if (previous) while (shared < Math.min(previous.bytes.length, inputState.bytes.length) && previous.bytes[shared] === inputState.bytes[shared]) shared++;
-    const unchanged = previous === null ? null : JSON.stringify(inputState.messages.slice(0, previous.messages.length)) === JSON.stringify(previous.messages);
-    previous = inputState;
-    ledger.charged += reservation;
-    const row: GatewayRecord = { kind: "reserved", request: ++requests, inputSha256: digest(wire), inputBytes: inputState.bytes.length,
-      sharedInputBytes: shared, prefixUnchanged: unchanged, hasTools: Array.isArray(payload["tools"]) && payload["tools"].length > 0,
-      reservation, charge: reservation, measured: null, ledgerCharged: ledger.charged, usage: null, failure: null, responseModel: null, wallMs: 0 };
-    const state = { row, started: performance.now(), terminal: null as NativeResponse | null, httpStatus: null as number | null };
-    current = state;
-    await options.record(row); await options.wire(payload, requests);
-    const response = await (options.fetch ?? fetch)(new Request(request, { redirect: "error", signal: AbortSignal.any([controller.signal, request.signal]) }));
-    state.httpStatus = response.status;
-    if (!response.ok || !response.body) { await response.body?.cancel(); throw Error("native upstream response failed"); }
-    return new Response(nativeResponseTap(response.body, async terminal => {
-      state.terminal = terminal; await options.response(terminal, row.request);
-    }), { status: response.status, headers: response.headers });
-  }) as typeof fetch;
-  const actual = new PiProvider(config, options.key, { oauth: true, effects: noEffects, fetch: observedFetch,
-    ...(options.sessionId ? { sessionId: options.sessionId } : {}), retry: { maxRetries: 0, maxRetryDelayMs: 0 } });
-  const provider: Provider = { config, async send(frozen, messages, node, signal, onDelta) {
-    if (active) throw Error("native eval model requests must be sequential");
-    active = true; current = undefined;
-    let finish = () => {}; idle = new Promise<void>(resolve => { finish = resolve; });
-    let failed = false;
-    try {
-      const turn = await actual.send(frozen, messages, node, AbortSignal.any([controller.signal, ...(signal ? [signal] : [])]), onDelta);
-      const state = current as { terminal: NativeResponse | null } | undefined;
-      const observed = nativeUsage(state?.terminal);
-      const same = observed && Object.entries(observed).every(([key, count]) => turn.usage[key as keyof typeof observed] === count);
-      const model = state?.terminal?.model;
-      if (!same || typeof model !== "string" || !(model === config.spec.id || new RegExp("^" + config.spec.id + "-\\d{4}-\\d{2}-\\d{2}$").test(model)))
-        throw Error("native model or usage differs from the production adapter");
-      return turn;
-    } catch {
-      failed = true; stopped ??= "error";
-      throw Error("native eval request failed; inspect status and retained usage");
-    } finally {
-      try {
-        const state = current as { row: GatewayRecord; started: number; terminal: NativeResponse | null; httpStatus: number | null } | undefined;
-        if (state) {
-          const usage = nativeUsage(state.terminal), measured = nativeMeasured(state.terminal, price), charge = measured ?? state.row.reservation;
-          ledger.charged += charge - state.row.reservation;
-          if (failed || measured === null || charge > state.row.reservation) { ledger.halted = true; stopped ??= "error"; }
-          await options.record({ ...state.row, kind: "settled", usage, measured, charge, ledgerCharged: ledger.charged,
-            responseModel: typeof state.terminal?.model === "string" ? state.terminal.model : null,
-            failure: failed ? { kind: [402, 429].includes(state.httpStatus ?? 0) ? "balance" : "provider", status: state.httpStatus } : null,
-            wallMs: performance.now() - state.started });
-        }
-      } catch { ledger.halted = true; throw Error("native eval ledger could not be persisted"); }
-      finally { active = false; finish(); }
-    }
-  } };
-  return { provider, get requests() { return requests; }, get stopped() { return stopped; }, async close() { controller.abort(); await idle; } };
-}

package/src/runtime/query-eval-verifier.ts

@@ -1,28 +0,0 @@
-/** Q.21: emit the isolated Bun verifier; it owns file I/O and process exit. */
-import type { queryEvalContract } from "../core/query-eval-contract.ts";
-
-export function queryEvalVerifier(contract: ReturnType<typeof queryEvalContract>): string {
-  // The standalone fixture carries only shape expectations and a one-way oracle
-  // digest. Diagnostics never expose expected numeric values to the model.
-  return `import { createHash } from "node:crypto";
-const paths = ${JSON.stringify(contract.paths)}, columns = ${JSON.stringify(contract.columns)}, rowCount = ${contract.rowCount};
-const fail = (reason, detail = {}) => { console.error(JSON.stringify({ok:false, reason, ...detail})); process.exit(1); };
-const body = await Bun.file("answer.txt").text().catch(() => fail("answer_missing"));
-let value; try { value = JSON.parse(body); } catch { fail("invalid_json"); }
-if (!Array.isArray(value) || value.length !== paths.length) fail("source_count", {expected:paths.length, actual:Array.isArray(value)?value.length:null});
-for (let i = 0; i < paths.length; i++) {
-  const item = value[i];
-  if (!item || typeof item !== "object" || JSON.stringify(Object.keys(item)) !== '["path","rows"]') fail("item_columns", {index:i, expected:["path","rows"]});
-  if (item.path !== paths[i]) fail("source_order", {index:i, expected:paths[i]});
-  if (!Array.isArray(item.rows) || item.rows.length !== rowCount) fail("row_count", {path:paths[i], expected:rowCount, actual:Array.isArray(item.rows)?item.rows.length:null});
-  for (let row = 0; row < item.rows.length; row++) {
-    const record = item.rows[row];
-    if (!record || typeof record !== "object" || JSON.stringify(Object.keys(record)) !== JSON.stringify(columns)) fail("result_columns", {path:paths[i], row, expected:columns, actual:record&&typeof record==="object"?Object.keys(record):null});
-    if (columns.some(column => column === "team" ? typeof record[column] !== "string" : !Number.isFinite(record[column]))) fail("result_type", {path:paths[i], row});
-  }
-}
-if (body !== JSON.stringify(value) + "\\n") fail("serialization", {expected:"compact JSON with one trailing newline"});
-if (createHash("sha256").update(body).digest("hex") !== ${JSON.stringify(contract.answerSha256)}) fail("value_mismatch", {next:"Recheck the original task, source rows, filter and aggregation. The result shape is correct; at least one value or row order differs."});
-console.log(JSON.stringify({ok:true}));
-`;
-}

package/src/runtime/recall-trial.ts

@@ -1,226 +0,0 @@
-/**
- * F2 delayed recall across three harnesses behind one metered gateway.
- *
- * Each page is deleted when its stage ends, so a later question can only be
- * answered from what the harness itself retained. The arms differ in exactly
- * that: xeno keeps a lossless log with a `history` reader; the ablation keeps
- * the same windowed projection without the reader; Pi keeps its own compaction
- * summary. See docs/tasks/bench-recall-runner.md.
- */
-import { appendFile, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
-import { dirname, join, relative } from "node:path";
-import { createHash } from "node:crypto";
-import { recallCodes, recallPrompt, verifyRecallAnswer, type RecallFixture, type RecallKind, type RecallVerdict } from "../core/recall-trial.ts";
-import type { BenchmarkPrice, BenchmarkUsage } from "../core/benchmark.ts";
-import { Session } from "./loop.ts";
-import { OpenAICompletionsProvider } from "./openai.ts";
-import type { ProviderConfig, ToolSchema } from "./provider.ts";
-import { COMPARISON_SYSTEM, COMPARISON_TOOLS, comparisonGateway, runPiPrompt, type ComparisonBackend, type ComparisonLedger, type GatewayRecord, type PiIdentity } from "./harness-comparison.ts";
-import { DEFAULT_TOOL_CONTEXT, TOOL_SCHEMAS, runTool } from "./tools.ts";
-
-export const RECALL_ARMS = ["xeno", "xeno-nohistory", "pi"] as const;
-export type RecallArm = typeof RECALL_ARMS[number];
-
-/** Native memory tools are the independent variable; everything else is shared. */
-export function recallToolNames(arm: RecallArm): readonly string[] {
-  return arm === "xeno" ? [...COMPARISON_TOOLS, "history", "note"] : COMPARISON_TOOLS;
-}
-export const recallTools = (arm: RecallArm): ToolSchema[] =>
-  recallToolNames(arm).map((name) => TOOL_SCHEMAS.find((tool) => tool.name === name)!);
-
-export interface RecallStageResult {
-  stage: number; kind: RecallKind; source: number; distance: number;
-  /** A context boundary (xeno cut / Pi compaction) separated the source page from the answer. */
-  crossed: boolean;
-  /**
-   * The expected code was in the first request of this stage. A crossed
-   * boundary does not imply loss (xeno retains recent exchanges, a Pi summary
-   * may copy a code); this measures loss on the wire, identically per arm.
-   */
-  visibleAtAsk: boolean | null;
-  /** Visible in every task request of the stage; a mid-stage boundary can evict it after the ask. Null for recent. */
-  visibleThroughout: boolean | null;
-  status: "answered" | "stopped" | "error"; reason: string; verdict: RecallVerdict;
-  requests: number; pageReads: number; historyCalls: number; boundariesAfter: number;
-}
-export interface RecallTrialResult {
-  arm: RecallArm; planned: number; unrun: number; stages: RecallStageResult[];
-  toolNames: string[]; boundaries: number; requests: number;
-  knownCost: number; reservedUnknown: number; usage: BenchmarkUsage; peakPrompt: number;
-  /** Files other than answers the model created: its own notes, reported separately. */
-  modelFiles: string[];
-  /** Workspace copies of deleted pages; must be empty for the stage to count as observed-once. */
-  leakedPages: string[];
-  stopped: "budget" | "balance" | "error" | "limit" | null; wallMs: number;
-}
-
-const sha = (body: string) => createHash("sha256").update(body).digest("hex");
-
-async function listFiles(root: string): Promise<string[]> {
-  const out: string[] = [];
-  for (const entry of await readdir(root, { withFileTypes: true, recursive: true })) {
-    if (entry.isFile()) out.push(relative(root, join(entry.parentPath, entry.name)));
-  }
-  return out.sort();
-}
-
-export async function runRecallTrial(options: {
-  directory: string; fixture: RecallFixture; arm: RecallArm;
-  config: ProviderConfig; price: BenchmarkPrice; ledger: ComparisonLedger; backend: ComparisonBackend;
-  maxTurnsPerStage: number; stageTimeoutMs: number;
-  pi: PiIdentity | null; extension: string;
-  /** Synthetic pressure only; live trials keep Pi's default compaction. */
-  piCompaction?: { reserveTokens: number; keepRecentTokens: number };
-  onProgress?: (event: Record<string, unknown>) => Promise<void>;
-}): Promise<RecallTrialResult> {
-  const { fixture, arm } = options, started = performance.now();
-  const workspace = join(options.directory, "workspace"), outputStore = join(options.directory, "observations");
-  await mkdir(workspace, { recursive: true }); await mkdir(outputStore, { recursive: true });
-  const tools = recallTools(arm), allow = recallToolNames(arm);
-  const result: RecallTrialResult = { arm, planned: fixture.pages.length, unrun: fixture.pages.length, stages: [],
-    toolNames: [...allow], boundaries: 0, requests: 0, knownCost: 0, reservedUnknown: 0,
-    usage: { input_tokens: 0, output_tokens: 0, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }, peakPrompt: 0,
-    modelFiles: [], leakedPages: [], stopped: null, wallMs: 0 };
-  let stage = 0, pageReads = 0, historyCalls = 0, stageTaskRequests = 0;
-  let seen: boolean[] = [];
-  const page = () => fixture.pages[stage]!;
-  const countCall = (name: string, input: unknown) => {
-    if (name === "history") historyCalls++;
-    if (name === "read" && input !== null && typeof input === "object" && (input as Record<string, unknown>)["path"] === page().path) pageReads++;
-  };
-  const toolContext = { ...DEFAULT_TOOL_CONTEXT, root: workspace, allow, outputStore };
-  const gateway = comparisonGateway({ config: options.config, price: options.price, ledger: options.ledger, tools,
-    // Pi's summary calls share this ceiling; the slack keeps them from starving task turns.
-    maxRequests: (options.maxTurnsPerStage + 2) * fixture.pages.length,
-    // The same per-stage cap for every arm: Session enforces maxTurns itself,
-    // Pi has no such flag, so the gateway refuses its excess task requests.
-    admit: (hasTools) => !hasTools || stageTaskRequests++ < options.maxTurnsPerStage,
-    backend: (payload, signal) => {
-      // Summary calls carry no tools; only a task turn shows what the model sees.
-      if (Array.isArray(payload["tools"])) seen.push(JSON.stringify(payload["messages"]).replaceAll("\\", "").includes(page().question.expected));
-      return options.backend(payload, signal);
-    },
-    executeTool: (name, input) => { countCall(name, input); return runTool(toolContext, name, input); },
-    record: async (record: GatewayRecord) => {
-      await appendFile(join(options.directory, "requests.jsonl"), JSON.stringify({ stage, ...record }) + "\n");
-      if (record.kind === "settled") {
-        result.requests++;
-        if (record.measured === null) result.reservedUnknown += record.charge; else result.knownCost += record.measured;
-        if (record.usage) {
-          for (const key of Object.keys(result.usage) as (keyof BenchmarkUsage)[]) result.usage[key] += record.usage[key];
-          result.peakPrompt = Math.max(result.peakPrompt, record.usage.input_tokens + record.usage.cache_read_input_tokens);
-        }
-      }
-      await options.onProgress?.({ phase: record.kind, arm, stage, request: record.request, measured: record.measured, failure: record.failure });
-    } });
-  const config = { ...options.config, spec: { ...options.config.spec, baseUrl: gateway.url } };
-  const codes = recallCodes(fixture), boundaryAfter: number[] = [];
-  let session: Session | null = null;
-  try {
-    if (arm !== "pi") {
-      await mkdir(join(options.directory, "log"));
-      const provider = new OpenAICompletionsProvider(config, gateway.token, { retry: { maxRetries: 0, maxRetryDelayMs: 0 } });
-      session = new Session({ logDir: join(options.directory, "log"), frozen: { system: COMPARISON_SYSTEM, tools },
-        tools: toolContext, contextPolicy: "windowed", maxTurns: options.maxTurnsPerStage, harnessReviews: false }, provider);
-      await session.start();
-    } else if (!options.pi) throw new Error("Pi identity required");
-    for (const current of fixture.pages) {
-      if (options.ledger.halted || gateway.stopped) break;
-      stage = current.stage; pageReads = 0; historyCalls = 0; seen = []; stageTaskRequests = 0;
-      const file = join(workspace, current.path);
-      await mkdir(dirname(file), { recursive: true }); await mkdir(join(workspace, "answers"), { recursive: true });
-      await writeFile(file, current.body, { flag: "wx" });
-      const requestsBefore = result.requests, signal = AbortSignal.timeout(options.stageTimeoutMs);
-      let status: RecallStageResult["status"] = "stopped", reason = "no completion";
-      try {
-        if (session) {
-          for await (const event of session.send(recallPrompt(fixture, stage), signal)) {
-            if (event.kind === "cut") result.boundaries++;
-            if (event.kind === "tool_call" && event.tool) countCall(event.tool.name, event.tool.input);
-            if (event.kind === "done") reason = event.reason ?? "unknown completion";
-          }
-          if (["stop", "end_turn"].includes(reason)) status = "answered";
-        } else {
-          const run = await runPiPrompt({ directory: options.directory, workspace, prompt: recallPrompt(fixture, stage), config,
-            gateway, price: options.price, pi: options.pi!, extension: options.extension, signal, resume: stage > 0, tools,
-            ...(options.piCompaction ? { piCompaction: options.piCompaction } : {}) });
-          result.boundaries += run.cuts; reason = run.reason; status = run.answered ? "answered" : "error";
-        }
-      } catch {
-        status = "error"; reason = signal.aborted ? "stage deadline" : gateway.stopped ? `model gateway: ${gateway.stopped}` : "stage failed";
-      }
-      boundaryAfter[stage] = result.boundaries;
-      const answer = await readFile(join(workspace, current.answerPath), "utf8").catch(() => null);
-      const verdict = verifyRecallAnswer(answer, current.question, codes);
-      await rm(file);
-      const { question } = current;
-      const row: RecallStageResult = { stage, kind: question.kind, source: question.source, distance: stage - question.source,
-        crossed: question.source < stage && result.boundaries > boundaryAfter[question.source]!,
-        visibleAtAsk: question.source === stage || seen.length === 0 ? null : seen[0]!,
-        visibleThroughout: question.source === stage || seen.length === 0 ? null : seen.every(Boolean),
-        status, reason, verdict, requests: result.requests - requestsBefore, pageReads, historyCalls, boundariesAfter: result.boundaries };
-      result.stages.push(row); result.unrun = result.planned - result.stages.length;
-      await writeFile(join(options.directory, `stage-${String(stage).padStart(3, "0")}.json`),
-        JSON.stringify({ ...row, answer }, null, 2) + "\n", { flag: "wx" });
-      await options.onProgress?.({ phase: "stage", arm, stage, verdict: verdict.reason, crossed: row.crossed, boundaries: result.boundaries });
-    }
-  } finally { await gateway.close(); }
-  const files = await listFiles(workspace), bodies = new Set(fixture.pages.map((p) => sha(p.body)));
-  result.modelFiles = files.filter((path) => !path.startsWith("answers/") && !path.startsWith("incoming/"));
-  for (const path of files) if (bodies.has(sha(await readFile(join(workspace, path), "utf8")))) result.leakedPages.push(path);
-  result.stopped = gateway.stopped; result.wallMs = performance.now() - started;
-  await writeFile(join(options.directory, "result.json"), JSON.stringify(result, null, 2) + "\n", { flag: "wx" });
-  return result;
-}
-
-/**
- * Mechanical backend: answers only from what the request actually shows.
- *
- * It cannot remember across requests, so a code it finds proves the
- * information was in that projection (or recovered by history). Its summary
- * text for Pi is invented here, which is why nothing it produces is a score.
- */
-export function recallMechanicalBackend(): ComparisonBackend {
-  let id = 0;
-  return async (payload) => {
-    const messages = payload["messages"] as Record<string, unknown>[], tools = payload["tools"] as { function: { name: string } }[] | undefined;
-    const reply = (content: string | null, call?: { name: string; input: unknown }) => ({
-      id: `recall-mock-${++id}`, object: "chat.completion", created: 1, model: "recall-mechanical",
-      choices: [{ index: 0, finish_reason: call ? "tool_calls" : "stop", message: { role: "assistant", content,
-        ...(call ? { tool_calls: [{ id: `call-${id}`, type: "function", function: { name: call.name, arguments: JSON.stringify(call.input) } }] } : {}) } }],
-      usage: { prompt_tokens: Math.ceil(JSON.stringify([tools ?? [], messages]).length / 3.6), completion_tokens: 64, prompt_cache_hit_tokens: 0 },
-    });
-    if (!tools) return reply("Checkpoint: staged pages processed and answers written.");
-    const text = (m: Record<string, unknown>) => typeof m["content"] === "string" ? m["content"] : JSON.stringify(m["content"] ?? "");
-    const at = messages.findLastIndex((m) => m["role"] === "user" && /Stage \d+: read /.test(text(m)));
-    // After a cut the current request can sit inside a handoff that also quotes
-    // earlier stages; the latest quoted stage is the active one.
-    const asks = [...text(messages[at]!).matchAll(/read (\S+) in full\. Question: what is the current code of ticket (T-\d{5})\? Write only the code to (\S+),/g)];
-    const [, path, ticket, answerPath] = asks.at(-1)!;
-    const calls = messages.slice(at + 1).flatMap((m) => (m["tool_calls"] as { function: { name: string; arguments: string } }[] | undefined) ?? [])
-      .map((c) => ({ name: c.function.name, input: JSON.parse(c.function.arguments) as Record<string, unknown> }));
-    if (calls.some((c) => c.name === "write")) return reply("Stage done.");
-    if (!calls.some((c) => c.name === "read" && c.input["path"] === path)) return reply(null, { name: "read", input: { path } });
-    // Backslashes vanish so nested JSON (history results) matches the same way.
-    const visible = JSON.stringify(messages).replaceAll("\\", "");
-    const pattern = new RegExp(`"ticket":"${ticket}","code":"(K-[0-9a-f]{12})"`, "g");
-    // Search results are newest first, so the first hit there is the latest code;
-    // in the conversation itself the last occurrence is the latest.
-    const search = messages.slice(at + 1).findLast((m) => m["role"] === "tool" && text(m).includes('"matches"'));
-    const found = (search ? [...text(search).replaceAll("\\", "").matchAll(pattern)][0]?.[1] : undefined)
-      ?? [...visible.matchAll(pattern)].at(-1)?.[1];
-    if (found) return reply(null, { name: "write", input: { path: answerPath, content: found } });
-    const names = new Set(tools.map((t) => t.function.name));
-    const history = calls.filter((c) => c.name === "history");
-    if (names.has("history") && history.length === 0) return reply(null, { name: "history", input: { query: ticket } });
-    if (names.has("history") && history.length === 1) {
-      // Pick an observation that read a page; the question prompt also matches the ticket.
-      const result = messages.slice(at + 1).findLast((m) => m["role"] === "tool" && text(m).includes('"matches"'));
-      let items: { event: string; tool?: string }[] = [];
-      try { items = (JSON.parse(text(result ?? {})) as { items: typeof items }).items; } catch { /* not a search page */ }
-      const event = items.find((item) => item.tool === "read")?.event;
-      if (event) return reply(null, { name: "history", input: { event } });
-    }
-    return reply(null, { name: "write", input: { path: answerPath, content: "UNKNOWN" } });
-  };
-}

package/src/runtime/recovery-coverage.ts

@@ -1,71 +0,0 @@
-/** R12c: collect all started trials, including crash prefixes, without provider access. */
-import { join, resolve } from "node:path";
-import { requestAccounting, summarizeRecoveryCoverage, type TrialCoverage } from "../core/evaluation-coverage.ts";
-import { deriveRecoveryMetrics } from "../core/recovery-evaluation.ts";
-import type { EvalResult } from "../core/experiment.ts";
-import { isTrialLifecycle } from "../core/trial-lifecycle.ts";
-import { readCursor } from "../store/log.ts";
-import { readNodeHistory } from "../store/restore.ts";
-import { auditEvalJob, auditEvalRequests, evalDirectory, type EvalPlan, type EvalEnd } from "./harness-eval.ts";
-import type { GatewayRecord } from "./harness-comparison.ts";
-import { verifiedRecoveryRetrievals } from "./recovery-evidence.ts";
-
-export function requestPrefix(body: string): { records: GatewayRecord[]; tail: "complete" | "truncated" } {
-  const chunks = body.split("\n"), records: GatewayRecord[] = [];
-  let tail: "complete" | "truncated" = "complete";
-  for (const [index, line] of chunks.entries()) {
-    if (!line && index === chunks.length - 1) continue;
-    try { records.push(JSON.parse(line) as GatewayRecord); }
-    catch {
-      if (index !== chunks.length - 1) throw Error("corrupt request ledger interior");
-      tail = "truncated";
-    }
-  }
-  return { records, tail };
-}
-
-export async function inspectRecoveryTrials(output: string, plan: EvalPlan, audit = false) {
-  if (![2, 5].includes(plan.version)) throw Error("complete coverage needs protocol v2 or v5");
-  const trials: TrialCoverage[] = [], results: EvalResult[] = [];
-  let charged = 0, halted = false;
-  for (const job of plan.jobs) {
-    const directory = evalDirectory(output, job), startPath = join(directory, "trial-start.json");
-    const requestsPath = join(directory, "requests.jsonl"), resultPath = join(directory, "result.json"), endPath = join(directory, "end.json");
-    if (!await Bun.file(startPath).exists()) {
-      if (await Bun.file(requestsPath).exists() || await Bun.file(resultPath).exists() || await Bun.file(endPath).exists()) throw Error("trial evidence lacks start identity");
-      continue;
-    }
-    const start = await Bun.file(startPath).json() as { job: string; ledgerBefore: number };
-    if (start.job !== job.id || start.ledgerBefore !== charged || halted) throw Error("global recovery ledger sequence changed");
-    const prefix = requestPrefix(await Bun.file(requestsPath).exists() ? await Bun.file(requestsPath).text() : "");
-    const end = await Bun.file(endPath).exists() ? await Bun.file(endPath).json() as EvalEnd : undefined;
-    if (end?.lifecycle && !isTrialLifecycle(end.lifecycle)) throw Error("invalid recovery lifecycle");
-    const accounting = requestAccounting(prefix.records, start.ledgerBefore);
-    if (end && (end.ledgerBefore !== start.ledgerBefore || end.ledgerAfter !== accounting.charged || prefix.tail !== "complete")) throw Error("recovery end differs from request ledger");
-    let result: EvalResult | undefined;
-    if (await Bun.file(resultPath).exists()) {
-      if (!end || prefix.tail !== "complete" || accounting.unsettledRequests) throw Error("final result lacks settled source");
-      result = audit ? await auditEvalJob(directory, plan, job) : await Bun.file(resultPath).json() as EvalResult;
-      if (result.id !== job.id) throw Error("result identity changed");
-      results.push(result);
-    } else if (audit) await auditEvalRequests(directory, plan, job, { records: prefix.records, ledgerBefore: start.ledgerBefore, allowPending: true });
-    let partialMetrics: EvalResult["recovery"];
-    // During request callbacks the cursor can be behind the active append; only
-    // an offline audit reads incomplete tool metrics and labels its outcome unknown.
-    if (audit && !result) {
-      const cursor = await readCursor(join(directory, "log"));
-      if (cursor) {
-        const history = await readNodeHistory(join(directory, "log"), cursor.node, undefined, "context");
-        if (!history.ok) throw Error(history.error);
-        const originals = await verifiedRecoveryRetrievals(directory, history.nodes, plan.limits.maxOutputBytes!);
-        partialMetrics = deriveRecoveryMetrics(history.nodes, new Set(originals.keys()), null, path => resolve(directory, "workspace", path));
-      }
-    }
-    trials.push({ id: job.id, accounting, tail: prefix.tail, ...(result ? { result } : {}),
-      ...(partialMetrics ? { partialMetrics } : {}), interruption: end?.lifecycle?.interruption?.source ?? null });
-    charged = accounting.charged;
-    halted = end === undefined || end.ledgerHalted === true || !accounting.fullyMeasured || prefix.tail === "truncated" ||
-      prefix.records.some(row => row.kind === "settled" && (row.failure !== null || row.measured !== null && row.measured > row.reservation));
-  }
-  return { trials, results, ledger: { budget: plan.budget, charged, halted }, summary: summarizeRecoveryCoverage(plan.jobs, trials, plan.mechanicalOnly) };
-}

package/src/runtime/recovery-evidence.ts

@@ -1,76 +0,0 @@
-/** R12c: offline evidence readers verify source bytes and never rerun commands. */
-import { join } from "node:path";
-import { createHash } from "node:crypto";
-import type { LogEvent } from "../core/events.ts";
-import { historyEntries } from "../core/history-query.ts";
-import { readObservation } from "../store/observations.ts";
-import { isRuntimeTargetReceipt } from "../core/runtime-target.ts";
-
-export function recoveryWorkEvents(events: readonly LogEvent[], last: string | undefined): LogEvent[] {
-  const at = events.findIndex(event => event.id === last);
-  if (at < 0 || events.slice(at + 1).some(event => !["process_exit", "target_observation"].includes(event.t)))
-    throw Error("invalid recovery work boundary");
-  return events.slice(0, at + 1);
-}
-const equal = (left: unknown, right: unknown) => JSON.stringify(left) === JSON.stringify(right);
-const originalFields = (entry: unknown) => {
-  if (!entry || typeof entry !== "object" || Array.isArray(entry)) return entry;
-  // Consumption-time freshness is metadata around the immutable event, not
-  // part of the original observation whose bytes this metric verifies.
-  const { environment: _freshness, ...original } = entry as Record<string, unknown>;
-  return original;
-};
-export async function verifiedRecoveryRetrievals(directory: string, nodes: readonly { node: number; events: readonly LogEvent[] }[], maxOutputBytes: number) {
-  const result = new Map<string, { source: string; text: string }>();
-  const calls = new Map<string, { name: string; input: unknown }>();
-  const prefix: { node: number; events: LogEvent[] }[] = [];
-  for (const part of nodes) {
-    const seen: LogEvent[] = []; prefix.push({ node: part.node, events: seen });
-    for (const event of part.events) {
-      if (event.t === "assistant") for (const block of event.blocks) if (block.type === "tool_use") calls.set(`${part.node}:${block.call_id}`, block);
-      if (event.t === "tool_invocation") calls.set(`${part.node}:${event.call_id}`, { name: event.name, input: event.input ?? calls.get(`${part.node}:${event.call_id}`)?.input });
-      if (event.t === "tool_result" && event.ok) {
-        const call = calls.get(`${event.invocation_ref?.node ?? part.node}:${event.call_id}`);
-        const input = call?.input as Record<string, unknown> | undefined;
-        if (call?.name === "history" && input) {
-          if (typeof input["artifact"] === "string") {
-            const page = await readObservation(join(directory, "observations"), input["artifact"], Number(input["offset"] ?? 0), Math.max(1, Math.floor(maxOutputBytes / 3)));
-            // Only bytes actually delivered to the model count; an archived but
-            // clipped response still requires another retrieval by the agent.
-            let delivered: unknown;
-            try { delivered = JSON.parse(event.content); }
-            catch {
-              // JSON escaping can make even a bounded byte page exceed the
-              // preview. Verify its archive, but do not credit unseen bytes.
-              if (!event.content_ref) throw Error("unreadable recovery artifact receipt");
-              const stored = await Bun.file(event.content_ref.path).text();
-              if (createHash("sha256").update(stored).digest("hex") !== event.content_ref.sha256 || !equal(JSON.parse(stored), page))
-                throw Error("recovery artifact archive differs from original bytes");
-            }
-            if (delivered !== undefined && !equal(delivered, page)) throw Error("recovery artifact receipt differs from original bytes");
-            if (delivered !== undefined && page.base64) result.set(`${part.node}:${event.id}`, { source: page.sha256, text: page.text });
-          } else if (typeof input["event"] === "string" || Array.isArray(input["events"])) {
-            const entries = historyEntries(prefix), ids = typeof input["event"] === "string" ? [input["event"]] : input["events"] as unknown[];
-            const originals = ids.map(id => entries.find(entry => entry.event === id));
-            let delivered: unknown;
-            try { delivered = JSON.parse(event.content); } catch { /* a clipped index is not a complete original event */ }
-            const batch = delivered as { items?: unknown[]; missing?: unknown[] } | undefined;
-            const actual = typeof input["event"] === "string" ? [delivered] : Array.isArray(batch?.items) && equal(batch?.missing, []) ? batch.items : [];
-            if (originals.every(Boolean) && equal(actual.map(originalFields), originals)) result.set(`${part.node}:${event.id}`,
-              { source: originals.map(entry => entry!.event).join(","), text: originals.map(entry => entry!.text).join("\n") });
-          }
-        }
-      }
-      if (event.t === "target_observation" && event.observation.state === "ready") {
-        const receipt = event.observation;
-        if (!isRuntimeTargetReceipt(receipt) || !receipt.manifest) throw Error("invalid target receipt");
-        const body = await Bun.file(receipt.manifest.path).text();
-        if (createHash("sha256").update(body).digest("hex") !== receipt.manifest.sha256) throw Error("target observation source changed");
-        const archived = JSON.parse(body) as { observation: unknown }, { manifest: _manifest, ...original } = receipt;
-        if (!equal(archived.observation, original)) throw Error("target observation differs from archived capture");
-      }
-      seen.push(event);
-    }
-  }
-  return result;
-}