@visual-z/xeno 0.7.2 → 0.8.0

package/src/core/harness-comparison.ts

@@ -1,90 +0,0 @@
-/** Shared evaluation protocol, not a production context policy (research N.10). */
-import { shuffled } from "./experiment.ts";
-import { canReserve, type BenchmarkUsage } from "./benchmark.ts";
-
-export const COMPARISON_ARMS = ["xeno-windowed", "xeno-continuous", "pi"] as const;
-export type ComparisonArm = typeof COMPARISON_ARMS[number];
-export interface ComparisonJob { id: string; arm: ComparisonArm }
-
-/** Null means an explicit provider-balance authorization, never zero allowance. */
-export function comparisonCanReserve(charged: number, reservation: number, budget: number | null): boolean {
-  if (![charged, reservation, charged + reservation].every(Number.isFinite) || charged < 0 || reservation <= 0) return false;
-  return budget === null || canReserve(charged, reservation, budget);
-}
-
-/** DeepSeek documents 402 as insufficient balance; other errors need their own diagnosis. */
-export function comparisonFailureKind(status: number | null): "balance" | "provider" {
-  return status === 402 ? "balance" : "provider";
-}
-
-export function comparisonSchedule(ids: readonly string[], seed: number): ComparisonJob[] {
-  if (ids.length === 0 || new Set(ids).size !== ids.length) throw new Error("empty or duplicate comparison tasks");
-  const arms = shuffled(COMPARISON_ARMS, seed);
-  return [...ids].sort().flatMap((id, i) => arms.map((_, position) => ({ id, arm: arms[(i + position) % arms.length]! })));
-}
-
-export function jsonObject(value: unknown): Record<string, unknown> {
-  if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("expected JSON object");
-  return value as Record<string, unknown>;
-}
-
-/** Missing cache usage is unknown, never a fabricated zero or a discounted bill. */
-export function comparisonUsage(body: unknown): BenchmarkUsage | null {
-  try {
-    const usage = jsonObject(jsonObject(body)["usage"]);
-    const prompt = usage["prompt_tokens"], output = usage["completion_tokens"];
-    const details = usage["prompt_tokens_details"];
-    const vendorCached = usage["prompt_cache_hit_tokens"];
-    const standardCached = details ? jsonObject(details)["cached_tokens"] : undefined;
-    if (vendorCached !== undefined && standardCached !== undefined && vendorCached !== standardCached) return null;
-    const cached = vendorCached ?? standardCached;
-    if (![prompt, output, cached].every((n) => typeof n === "number" && Number.isSafeInteger(n) && n >= 0)) return null;
-    const p = prompt as number, o = output as number, c = cached as number;
-    if (p === 0 || o === 0 || c > p) return null;
-    if (usage["prompt_cache_miss_tokens"] !== undefined && usage["prompt_cache_miss_tokens"] !== p - c) return null;
-    return { input_tokens: p - c, output_tokens: o, cache_read_input_tokens: c, cache_creation_input_tokens: 0 };
-  } catch { return null; }
-}
-
-/** Buffer upstream responses for one billing boundary; do not report this as TTFT. */
-export function completionSse(value: unknown): string {
-  const body = jsonObject(value);
-  const choices = body["choices"];
-  if (!Array.isArray(choices) || choices.length !== 1) throw new Error("expected one completion");
-  const choice = jsonObject(choices[0]), message = jsonObject(choice["message"]);
-  const calls = message["tool_calls"];
-  const finish = choice["finish_reason"];
-  if (typeof finish !== "string" || !["stop", "tool_calls", "length", "content_filter"].includes(finish)) throw new Error("incomplete completion");
-  if (calls !== undefined && !Array.isArray(calls)) throw new Error("invalid tool calls");
-  const delta = { ...message, ...(Array.isArray(calls) ? { tool_calls: calls.map((call: unknown, index: number) => ({ ...jsonObject(call), index })) } : {}) };
-  const common = { id: body["id"], object: "chat.completion.chunk", created: body["created"], model: body["model"] };
-  return [
-    { ...common, choices: [{ index: 0, delta, finish_reason: null }] },
-    { ...common, choices: [{ index: 0, delta: {}, finish_reason: finish }] },
-    { ...common, choices: [], usage: body["usage"] },
-  ].map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n";
-}
-
-/** JSONL permits U+2028 in strings. Only LF frames an event, including Pi's output. */
-export function comparisonLines(buffer: string, chunk: string): { buffer: string; values: unknown[] } {
-  const lines = (buffer + chunk).split("\n");
-  const remainder = lines.pop()!;
-  return { buffer: remainder, values: lines.filter((line) => line.trim() !== "").map((line) => JSON.parse(line) as unknown) };
-}
-
-export function comparisonSummary(plan: readonly ComparisonJob[], results: readonly (ComparisonJob & { status: string; correct: boolean })[]) {
-  const keys = new Set(plan.map((job) => `${job.id}/${job.arm}`)), seen = new Set<string>();
-  if (keys.size !== plan.length) throw new Error("duplicate planned comparison job");
-  for (const result of results) {
-    const key = `${result.id}/${result.arm}`;
-    if (!keys.has(key) || seen.has(key) || (result.correct && result.status !== "answered")) throw new Error("invalid comparison outcome");
-    seen.add(key);
-  }
-  return COMPARISON_ARMS.map((arm) => {
-    const planned = plan.filter((job) => job.arm === arm).length, rows = results.filter((row) => row.arm === arm);
-    const correct = rows.filter((row) => row.correct).length;
-    return { arm, planned, attempted: rows.length, answered: rows.filter((row) => row.status === "answered").length,
-      correct, missing: planned - rows.length, accuracy: rows.length ? correct / rows.length : null,
-      complete: rows.length === planned };
-  }).filter((row) => row.planned > 0);
-}

package/src/core/harness-diagnostics.ts

@@ -1,206 +0,0 @@
-/** D11 / O.3: classify durable execution facts, never error words in tool output. */
-import type { LogEvent, ToolResultEvent } from "./events.ts";
-import type { BenchmarkUsage } from "./benchmark.ts";
-import { canonicalValue } from "./continuation.ts";
-
-export type ResultClass = "success" | "invalid_input" | "cancelled" | "timeout" | "command_nonzero" |
-  "execution_failed" | "rejected" | "unclassified_failure";
-
-export function classifyToolResult(event: ToolResultEvent): ResultClass {
-  const code = event.diagnostic?.code;
-  if (code === "invalid_input") return "invalid_input";
-  if (code === "cancelled" || event.outcome?.reason === "cancelled") return "cancelled";
-  if (code === "timeout" || event.outcome?.reason === "timeout") return "timeout";
-  if (event.effect?.kind === "command" && event.effect.exit !== 0) return "command_nonzero";
-  if (event.ok) return "success";
-  if (event.diagnostic?.execution === "not_started" ||
-    ["version_mismatch", "not_found", "ambiguous_edit", "permission_denied", "environment_missing"].includes(code ?? "")) return "rejected";
-  if (code === "execution_failed" || code === "unknown_effect" || event.outcome?.reason === "execution_error") return "execution_failed";
-  return "unclassified_failure";
-}
-
-export interface DiagnosticRequest {
-  request: number;
-  node: number | null;
-  usage: BenchmarkUsage | null;
-  responseModel: string | null;
-  failure: string | null;
-  firstDeltaMs: number | null;
-  wallMs: number | null;
-  prefixUnchanged: boolean | null;
-  sharedInputBytes: number | null;
-  frozenSha256: string | null;
-  /** Raw terminal counts can remain known when a cache breakdown is absent. */
-  reportedPromptTokens?: number | null;
-  reportedOutputTokens?: number | null;
-}
-export interface ToolCounters {
-  calls: number; providerCalls: number; invocations: number; nestedInvocations: number;
-  results: number; pairedCalls: number; orphanCalls: number; orphanResults: number;
-  duplicateProviderCalls: number; duplicateInvocations: number; duplicateResults: number;
-  repeatedProviderInputs: number; repeatedInvocationInputs: number;
-  classes: Record<ResultClass, number>;
-  diagnosticCodes: Record<string, number>;
-  commandExits: Record<string, number>;
-  expectedNonzeroCommands: number;
-  undeclaredNonzeroCommands: number;
-}
-const classes = (): Record<ResultClass, number> => ({ success: 0, invalid_input: 0, cancelled: 0, timeout: 0,
-  command_nonzero: 0, execution_failed: 0, rejected: 0, unclassified_failure: 0 });
-const counters = (): ToolCounters => ({ calls: 0, providerCalls: 0, invocations: 0, nestedInvocations: 0,
-  results: 0, pairedCalls: 0, orphanCalls: 0, orphanResults: 0, duplicateProviderCalls: 0,
-  duplicateInvocations: 0, duplicateResults: 0, repeatedProviderInputs: 0, repeatedInvocationInputs: 0,
-  classes: classes(), diagnosticCodes: {}, commandExits: {}, expectedNonzeroCommands: 0, undeclaredNonzeroCommands: 0 });
-function add(record: Record<string, number>, key: string): void { record[key] = (record[key] ?? 0) + 1; }
-const validCount = (value: number): boolean => Number.isSafeInteger(value) && value >= 0;
-const reportedCount = (value: number | null | undefined): number | null => typeof value === "number" && validCount(value) ? value : null;
-const validTime = (value: number | null): number | null => value !== null && Number.isFinite(value) && value >= 0 ? value : null;
-function observedUsage(usage: BenchmarkUsage | null): BenchmarkUsage | null {
-  return usage && [usage.input_tokens, usage.output_tokens, usage.cache_read_input_tokens,
-    usage.cache_creation_input_tokens].every(validCount) ? usage : null;
-}
-function quantile(values: readonly number[], q: number): number | null {
-  return values.length ? values.toSorted((a, b) => a - b)[Math.max(0, Math.ceil(values.length * q) - 1)]! : null;
-}
-interface Call {
-  name: string; provider: boolean; invocation: boolean; nested: boolean;
-  providerInput: string | null; invocationInput: string | null;
-}
-
-/**
- * Inputs and outputs remain private. Equal-input counts use canonical original
- * model arguments and immutable admitted-input hashes separately (O.3 / Pi A).
- * Expected nonzero exits need an evaluator declaration; an exit alone cannot
- * prove a command was accidental, and model prose cannot establish intent.
- */
-export function summarizeHarnessDiagnostics(input: {
-  events: readonly LogEvent[]; requests: readonly DiagnosticRequest[];
-  expectedNonzeroCallIds?: readonly string[];
-}) {
-  const tools: Record<string, ToolCounters> = {};
-  const tool = (name: string): ToolCounters => tools[name] ??= counters();
-  const calls = new Map<string, Call>(), results = new Map<string, ToolResultEvent[]>(), seenEvents = new Set<string>();
-  const guidance: Record<string, number> = {}, cuts: Record<string, number> = {}, cleanupProcessExits: Record<string, number> = {};
-  let duplicateEvents = 0, interruptions = 0;
-  for (const event of input.events) {
-    if (seenEvents.has(event.id)) { duplicateEvents++; continue; }
-    seenEvents.add(event.id);
-    if (event.t === "guidance") add(guidance, event.reason);
-    if (event.t === "cut") add(cuts, event.reason);
-    if (event.t === "interrupt") interruptions++;
-    if (event.t === "process_exit") add(cleanupProcessExits, String(event.exit));
-    if (event.t === "assistant") for (const block of event.blocks) {
-      if (block.type !== "tool_use") continue;
-      const existing = calls.get(block.call_id), stats = tool(block.name);
-      if (existing?.provider) stats.duplicateProviderCalls++;
-      calls.set(block.call_id, { name: existing?.name ?? block.name, provider: true,
-        invocation: existing?.invocation ?? false, nested: existing?.nested ?? false,
-        providerInput: canonicalValue(block.input), invocationInput: existing?.invocationInput ?? null });
-    }
-    if (event.t === "tool_invocation") {
-      const existing = calls.get(event.call_id), stats = tool(event.name);
-      if (existing?.invocation) stats.duplicateInvocations++;
-      calls.set(event.call_id, { name: existing?.name ?? event.name, provider: existing?.provider ?? false,
-        invocation: true, nested: event.parent_call_id !== undefined,
-        providerInput: existing?.providerInput ?? null, invocationInput: event.input_ref.sha256 });
-    }
-    if (event.t === "tool_result") {
-      const found = results.get(event.call_id) ?? [];
-      found.push(event); results.set(event.call_id, found);
-    }
-  }
-  const providerInputs = new Map<string, Set<string>>(), invocationInputs = new Map<string, Set<string>>();
-  const repeat = (groups: Map<string, Set<string>>, name: string, key: string | null): boolean => {
-    if (key === null) return false;
-    const seen = groups.get(name) ?? new Set<string>(), repeated = seen.has(key);
-    seen.add(key); groups.set(name, seen); return repeated;
-  };
-  for (const [id, call] of calls) {
-    const stats = tool(call.name); stats.calls++;
-    if (call.provider) stats.providerCalls++;
-    if (call.invocation) stats.invocations++;
-    if (call.nested) stats.nestedInvocations++;
-    if (repeat(providerInputs, call.name, call.providerInput)) stats.repeatedProviderInputs++;
-    if (repeat(invocationInputs, call.name, call.invocationInput)) stats.repeatedInvocationInputs++;
-    if (results.has(id)) stats.pairedCalls++; else stats.orphanCalls++;
-  }
-  const expected = new Set(input.expectedNonzeroCallIds ?? []);
-  for (const [id, found] of results) {
-    const call = calls.get(id), stats = tool(call?.name ?? "<orphan>");
-    stats.duplicateResults += Math.max(0, found.length - 1);
-    for (const event of found) {
-      stats.results++;
-      if (!call) stats.orphanResults++;
-      stats.classes[classifyToolResult(event)]++;
-      if (event.diagnostic) add(stats.diagnosticCodes, event.diagnostic.code);
-      if (event.effect?.kind === "command") {
-        add(stats.commandExits, String(event.effect.exit));
-        if (event.effect.exit !== 0) {
-          if (expected.has(id)) stats.expectedNonzeroCommands++; else stats.undeclaredNonzeroCommands++;
-        }
-      }
-    }
-  }
-  const requestIds = new Set<number>(), requests: {
-    request: number; node: number | null; usageKnown: boolean; promptTokens: number | null;
-    cachedTokens: number | null; cacheWriteTokens: number | null; outputTokens: number | null;
-    cacheHitRate: number | null; responseModel: string | null; failure: string | null;
-    firstDeltaMs: number | null; wallMs: number | null; prefixUnchanged: boolean | null; sharedInputBytes: number | null;
-  }[] = [];
-  const models: Record<string, number> = {};
-  const usage = { knownRequests: 0, unknownRequests: 0, requestsWithCacheReads: 0, promptTokens: 0, cachedTokens: 0,
-    cacheWriteTokens: 0, outputTokens: 0, promptKnownRequests: 0, promptUnknownRequests: 0,
-    cacheCoveredPromptTokens: 0, cacheCoverageRate: null as number | null,
-    observedCacheHitRate: null as number | null, cacheHitRate: null as number | null };
-  const prefix = { sameNodeComparisons: 0, sameNodeMutations: 0, boundaryComparisons: 0, boundaryMutations: 0,
-    unknownComparisons: 0, sameNodeFrozenMutations: 0, boundaryFrozenMutations: 0 };
-  let previous: DiagnosticRequest | undefined, duplicateRequests = 0;
-  for (const request of input.requests.toSorted((a, b) => a.request - b.request)) {
-    if (requestIds.has(request.request)) { duplicateRequests++; continue; }
-    requestIds.add(request.request);
-    const observed = observedUsage(request.usage);
-    const prompt = reportedCount(request.reportedPromptTokens) ?? (observed ?
-      observed.input_tokens + observed.cache_read_input_tokens + observed.cache_creation_input_tokens : null);
-    const output = reportedCount(request.reportedOutputTokens) ?? observed?.output_tokens ?? null;
-    if (prompt !== null) { usage.promptKnownRequests++; usage.promptTokens += prompt; } else usage.promptUnknownRequests++;
-    if (output !== null) usage.outputTokens += output;
-    if (observed) {
-      usage.knownRequests++; usage.cacheCoveredPromptTokens += prompt!; usage.cachedTokens += observed.cache_read_input_tokens;
-      usage.cacheWriteTokens += observed.cache_creation_input_tokens;
-      if (observed.cache_read_input_tokens > 0) usage.requestsWithCacheReads++;
-    } else usage.unknownRequests++;
-    if (request.responseModel !== null) add(models, request.responseModel);
-    if (previous) {
-      if (previous.node === null || request.node === null || request.prefixUnchanged === null) prefix.unknownComparisons++;
-      else if (previous.node === request.node) { prefix.sameNodeComparisons++; if (!request.prefixUnchanged) prefix.sameNodeMutations++; }
-      else { prefix.boundaryComparisons++; if (!request.prefixUnchanged) prefix.boundaryMutations++; }
-      if (previous.frozenSha256 !== null && request.frozenSha256 !== null && previous.frozenSha256 !== request.frozenSha256) {
-        if (previous.node === request.node && request.node !== null) prefix.sameNodeFrozenMutations++;
-        else if (previous.node !== null && request.node !== null) prefix.boundaryFrozenMutations++;
-      }
-    }
-    previous = request;
-    requests.push({ request: request.request, node: request.node, usageKnown: observed !== null, promptTokens: prompt,
-      cachedTokens: observed?.cache_read_input_tokens ?? null, cacheWriteTokens: observed?.cache_creation_input_tokens ?? null,
-      outputTokens: output, cacheHitRate: observed && prompt ? observed.cache_read_input_tokens / prompt : null,
-      responseModel: request.responseModel, failure: request.failure, firstDeltaMs: validTime(request.firstDeltaMs),
-      wallMs: validTime(request.wallMs), prefixUnchanged: request.prefixUnchanged, sharedInputBytes: request.sharedInputBytes });
-  }
-  usage.observedCacheHitRate = usage.cacheCoveredPromptTokens > 0 ? usage.cachedTokens / usage.cacheCoveredPromptTokens : null;
-  usage.cacheCoverageRate = usage.promptUnknownRequests === 0 && usage.promptTokens > 0 ? usage.cacheCoveredPromptTokens / usage.promptTokens : null;
-  usage.cacheHitRate = usage.unknownRequests === 0 && usage.promptTokens > 0 ? usage.cachedTokens / usage.promptTokens : null;
-  const delta = requests.flatMap(r => r.firstDeltaMs === null ? [] : [r.firstDeltaMs]);
-  const totals = counters();
-  for (const stats of Object.values(tools)) {
-    for (const key of Object.keys(totals) as (keyof ToolCounters)[]) {
-      const value = stats[key];
-      if (typeof value === "number") (totals[key] as number) += value;
-    }
-    for (const key of Object.keys(stats.classes) as ResultClass[]) totals.classes[key] += stats.classes[key];
-    for (const key of Object.keys(stats.commandExits)) totals.commandExits[key] = (totals.commandExits[key] ?? 0) + stats.commandExits[key]!;
-    for (const key of Object.keys(stats.diagnosticCodes)) totals.diagnosticCodes[key] = (totals.diagnosticCodes[key] ?? 0) + stats.diagnosticCodes[key]!;
-  }
-  return { eventCount: seenEvents.size, duplicateEvents, tools, totals, guidance, cuts, interruptions, cleanupProcessExits,
-    requestCount: requests.length, duplicateRequests, models, usage, prefix, requests,
-    firstDeltaMs: { samples: delta.length, missing: requests.length - delta.length, p50: quantile(delta, 0.5), p95: quantile(delta, 0.95) } };
-}

package/src/core/native-eval.ts

@@ -1,25 +0,0 @@
-/** Q.20: native subscription measurements are standard API price equivalents, never invoices. */
-import { measuredCharge, requestReservation, type BenchmarkPrice, type BenchmarkUsage } from "./benchmark.ts";
-
-// Verified Luna short-context tier. Bound wire bytes conservatively before dispatch;
-// crossing this tier needs a new protocol, not silent use of the cheaper price.
-export const NATIVE_EVAL_INPUT_LIMIT = 272000;
-export function nativeUsage(response: unknown): BenchmarkUsage | null {
-  if (!response || typeof response !== "object") return null;
-  const raw = (response as Record<string, unknown>)["usage"];
-  if (!raw || typeof raw !== "object") return null;
-  const u = raw as Record<string, unknown>, details = u["input_tokens_details"] as Record<string, unknown> | undefined;
-  const input = u["input_tokens"], output = u["output_tokens"], cached = details?.["cached_tokens"] ?? 0, created = details?.["cache_write_tokens"] ?? 0;
-  if (![input, output, cached, created].every(n => typeof n === "number" && Number.isSafeInteger(n) && n >= 0) ||
-      Number(input) <= 0 || Number(output) <= 0 || Number(input) > NATIVE_EVAL_INPUT_LIMIT || Number(cached) + Number(created) > Number(input)) return null;
-  return { input_tokens: Number(input) - Number(cached) - Number(created), output_tokens: Number(output),
-    cache_read_input_tokens: Number(cached), cache_creation_input_tokens: Number(created) };
-}
-export function nativeReservation(bytes: number, modelMaxOutput: number, price: BenchmarkPrice): number {
-  if (bytes + 4096 > NATIVE_EVAL_INPUT_LIMIT || price.cacheWrite === undefined) throw Error("native eval price tier or cache-write rate is not covered");
-  return requestReservation(bytes, modelMaxOutput, { ...price, input: Math.max(price.input, price.cacheWrite) });
-}
-export function nativeMeasured(response: unknown, price: BenchmarkPrice): number | null {
-  const usage = nativeUsage(response);
-  return usage && price.cacheWrite !== undefined ? measuredCharge(usage, price) : null;
-}

package/src/core/policy-evaluation.ts

@@ -1,55 +0,0 @@
-/** N.7: repetitions stay attached to their task; they are not independent questions. */
-import { type ContextPolicy } from "./context-policy.ts";
-import { shuffled } from "./experiment.ts";
-
-// N.7 is a frozen three-arm protocol; new production policies cannot change it.
-const EVALUATED_POLICIES: readonly ContextPolicy[] = ["segmented", "continuous", "adaptive"];
-
-export interface PolicyJob { id: string; policy: ContextPolicy; repeat: number }
-export function policyJobKey(job: PolicyJob): string { return JSON.stringify([job.id, job.policy, job.repeat]); }
-export function policySchedule(ids: readonly string[], repetitions: number, seed: number): PolicyJob[] {
-  if (ids.length === 0 || new Set(ids).size !== ids.length || !Number.isSafeInteger(repetitions) || repetitions < 1) throw new Error("invalid policy trial design");
-  const policies = shuffled(EVALUATED_POLICIES, seed);
-  const jobs: PolicyJob[] = [];
-  let block = 0;
-  for (let repeat = 0; repeat < repetitions; repeat++) for (const id of shuffled([...ids].sort(), seed + repeat + 1)) {
-    for (let position = 0; position < policies.length; position++) jobs.push({ id, repeat, policy: policies[(block + position) % policies.length]! });
-    block++;
-  }
-  return jobs;
-}
-export interface PolicyObservation extends PolicyJob {
-  status: string; correct: boolean; peakRateUsd: number; reservedUnknownUsd: number;
-  requests: number; cuts: number; incompleteHandoffs: number; wallMs: number;
-  usage: { prompt_tokens: number; cache_read_input_tokens: number; output_tokens: number };
-}
-export function summarizePolicies(plan: readonly PolicyJob[], results: readonly PolicyObservation[]) {
-  const planned = new Set(plan.map(policyJobKey)), observed = new Set<string>();
-  if (planned.size !== plan.length) throw new Error("duplicate planned policy job");
-  for (const result of results) {
-    const key = policyJobKey(result);
-    if (!planned.has(key) || observed.has(key) || (result.correct && result.status !== "answered")) throw new Error("invalid policy result");
-    observed.add(key);
-  }
-  return [...new Set(plan.map((job) => job.policy))].sort().map((policy) => {
-    const schedule = plan.filter((job) => job.policy === policy), group = results.filter((row) => row.policy === policy);
-    const sum = (read: (row: PolicyObservation) => number) => group.reduce((n, row) => n + read(row), 0);
-    const correct = group.filter((row) => row.correct).length;
-    const peakRateUsd = sum((row) => row.peakRateUsd), reservedUnknownUsd = sum((row) => row.reservedUnknownUsd);
-    const promptTokens = sum((row) => row.usage.prompt_tokens), cachedTokens = sum((row) => row.usage.cache_read_input_tokens);
-    return { policy, planned: schedule.length, attempted: group.length, missing: schedule.length - group.length,
-      independentTasks: new Set(schedule.map((job) => job.id)).size, correct,
-      accuracy: group.length ? correct / group.length : null, complete: group.length === schedule.length,
-      plannedSolvedFraction: schedule.length ? correct / schedule.length : null,
-      peakRateUsd, reservedUnknownUsd, allAttemptCostPerSuccessUsd: correct ? (peakRateUsd + reservedUnknownUsd) / correct : null,
-      promptTokens, cachedTokens, outputTokens: sum((row) => row.usage.output_tokens),
-      cacheHitRate: promptTokens ? cachedTokens / promptTokens : null,
-      requests: sum((row) => row.requests), cuts: sum((row) => row.cuts), incompleteHandoffs: sum((row) => row.incompleteHandoffs),
-      wallMs: sum((row) => row.wallMs),
-      tasks: [...new Set(schedule.map((job) => job.id))].map((id) => ({ id,
-        planned: schedule.filter((job) => job.id === id).length,
-        attempted: group.filter((row) => row.id === id).length,
-        correct: group.filter((row) => row.id === id && row.correct).length })),
-    };
-  });
-}

package/src/core/query-eval-contract.ts

@@ -1,14 +0,0 @@
-/** Q.21: task-specific shape advice cannot reveal the independent numeric oracle. */
-import type { ToolModeTask } from "./tool-mode-protocol.ts";
-
-export function queryEvalContract(task: ToolModeTask, paths: readonly string[], answerSha256: string) {
-  const columns = task === "tool-small" ? ["amount"] : task === "tool-batch" ? ["total"] : ["team", "total"];
-  const rowCount = task === "tool-aggregate" ? 4 : 1;
-  const operation = task === "tool-small" ? "select amount; return one row with only amount" : task === "tool-batch"
-    ? "sum amount across all records as total; return one row with only total per source"
-    : "keep enabled=true, group by team, sum amount as total, and sort by team ascending; return four rows with columns team,total in that order";
-  const instruction = "Use query on each of " + paths.join(", ") + " to " + operation +
-    '. Write compact JSON with a trailing newline to answer.txt: [{"path":"data-0.jsonl","rows":[...]}], one item per source in the listed order. ' +
-    "Preserve all fixture inputs and run bun run gate.";
-  return { instruction, columns, rowCount, paths: [...paths], answerSha256 };
-}

package/src/core/recall-analysis.ts

@@ -1,90 +0,0 @@
-/**
- * Pre-registered F2 analysis (docs/tasks/bench-recall-pilot.md). Pure.
- *
- * The unit is a (seed, ratio) block: arms in one block face byte-identical
- * pages and questions, blocks are independent, stages within a run are not.
- */
-export interface RecallStageRow {
-  stage: number; kind: "recent" | "recall" | "latest"; distance: number;
-  visibleThroughout: boolean | null; verdict: { ok: boolean; reason: string }; status: string;
-}
-export interface RecallRunRow {
-  arm: string; ratio: number; seed: number; planned: number; unrun: number; boundaries: number;
-  knownCost: number; reservedUnknown: number; stages: RecallStageRow[]; modelFiles: string[];
-}
-
-/** Two-sided exact sign test; ties are dropped before counting. */
-export function signTest(differences: readonly number[]): { positive: number; negative: number; ties: number; p: number } {
-  const positive = differences.filter((d) => d > 0).length, negative = differences.filter((d) => d < 0).length;
-  const n = positive + negative, k = Math.min(positive, negative);
-  let tail = 0;
-  for (let i = 0; i <= k; i++) tail += binomial(n, i) / 2 ** n;
-  return { positive, negative, ties: differences.length - n, p: n === 0 ? 1 : Math.min(1, 2 * tail) };
-}
-
-function binomial(n: number, k: number): number {
-  let out = 1;
-  for (let i = 1; i <= k; i++) out = out * (n - k + i) / i;
-  return out;
-}
-
-export function median(values: readonly number[]): number | null {
-  if (values.length === 0) return null;
-  const sorted = [...values].sort((a, b) => a - b), mid = Math.floor(sorted.length / 2);
-  return sorted.length % 2 ? sorted[mid]! : (sorted[mid - 1]! + sorted[mid]!) / 2;
-}
-
-/** Non-recent accuracy over planned questions: an unrun stage counts as wrong, never as missing. */
-export function memoryAccuracy(run: RecallRunRow, planned: readonly RecallStageRow["kind"][]): number | null {
-  const asked = planned.filter((kind) => kind !== "recent").length;
-  if (asked === 0) return null;
-  return run.stages.filter((row) => row.kind !== "recent" && row.verdict.ok).length / asked;
-}
-
-export interface BlockDifference { seed: number; ratio: number; a: number | null; b: number | null; difference: number | null }
-
-export function pairedBlocks(runs: readonly RecallRunRow[], a: string, b: string, ratios: readonly number[],
-  plannedKinds: (seed: number, ratio: number) => readonly RecallStageRow["kind"][]): BlockDifference[] {
-  const blocks = [...new Set(runs.filter((run) => ratios.includes(run.ratio)).map((run) => `${run.seed}/${run.ratio}`))].sort();
-  return blocks.map((key) => {
-    const [seed, ratio] = key.split("/").map(Number) as [number, number];
-    const kinds = plannedKinds(seed, ratio);
-    const find = (arm: string) => runs.find((run) => run.arm === arm && run.seed === seed && run.ratio === ratio);
-    const ra = find(a), rb = find(b);
-    const accA = ra ? memoryAccuracy(ra, kinds) : null, accB = rb ? memoryAccuracy(rb, kinds) : null;
-    return { seed, ratio, a: accA, b: accB, difference: accA === null || accB === null ? null : accA - accB };
-  });
-}
-
-export function compare(blocks: readonly BlockDifference[]) {
-  // A block missing either arm is reported, never silently imputed.
-  const complete = blocks.filter((block) => block.difference !== null).map((block) => block.difference!);
-  return { blocks: blocks.length, complete: complete.length, medianDifference: median(complete), ...signTest(complete) };
-}
-
-/** Pre-registered decision: p < 0.05 and median difference ≥ 10 points. */
-export function decide(primary: ReturnType<typeof compare>): "advantage" | "no_advantage_measured" {
-  return primary.complete > 0 && primary.p < 0.05 && (primary.medianDifference ?? 0) >= 0.10 ? "advantage" : "no_advantage_measured";
-}
-
-export function describeArm(runs: readonly RecallRunRow[], arm: string, ratio: number) {
-  const rows = runs.filter((run) => run.arm === arm && run.ratio === ratio);
-  const stages = rows.flatMap((run) => run.stages), memory = stages.filter((row) => row.kind !== "recent");
-  const evicted = memory.filter((row) => row.visibleThroughout === false);
-  const latest = stages.filter((row) => row.kind === "latest");
-  const correct = stages.filter((row) => row.verdict.ok).length;
-  const cost = rows.reduce((sum, run) => sum + run.knownCost, 0);
-  const rate = (n: number, d: number) => d === 0 ? null : n / d;
-  return {
-    arm, ratio, runs: rows.length, stages: stages.length, unrun: rows.reduce((n, run) => n + run.unrun, 0),
-    recentAccuracy: rate(stages.filter((row) => row.kind === "recent" && row.verdict.ok).length, stages.filter((row) => row.kind === "recent").length),
-    memoryAccuracy: rate(memory.filter((row) => row.verdict.ok).length, memory.length),
-    evictionRate: rate(evicted.length, memory.length),
-    evictedAccuracy: rate(evicted.filter((row) => row.verdict.ok).length, evicted.length),
-    staleLatest: rate(latest.filter((row) => row.verdict.reason === "stale").length, latest.length),
-    boundaries: rows.reduce((n, run) => n + run.boundaries, 0),
-    cost, costPerCorrect: correct === 0 ? null : cost / correct,
-    reservedUnknown: rows.reduce((n, run) => n + run.reservedUnknown, 0),
-    runsWithModelFiles: rows.filter((run) => run.modelFiles.length > 0).length,
-  };
-}

package/src/core/recall-trial.ts

@@ -1,153 +0,0 @@
-/**
- * F2 delayed recall: information observed exactly once, asked about later.
- *
- * The runner deletes each page when its stage ends, so the workspace holds no
- * durable copy; only what the harness kept (a log, a summary, the model's own
- * notes) can answer a later question. This isolates the claim the harness makes
- * -- lossless history beats lossy summaries when a detail's relevance is only
- * revealed later -- from the F1 ledger task, where state lives on disk and any
- * file-capable harness can continue. Modelled on MemoryAgentBench's incremental
- * chunk injection (accurate retrieval + conflict resolution); none of its data
- * or scoring is reused. Pure: no I/O, clock or ambient randomness.
- */
-import { seededRandom } from "./experiment.ts";
-
-export type RecallKind = "recent" | "recall" | "latest";
-export interface RecallEvent { ticket: string; code: string; action: "assign" | "reassign" }
-export interface RecallQuestion {
-  stage: number; kind: RecallKind; ticket: string; expected: string;
-  /** Page holding the expected code; distance is stage - source. */
-  source: number;
-  /** Codes the ticket held earlier; answering one of them is a stale answer. */
-  stale: string[];
-}
-export interface RecallPage { stage: number; path: string; answerPath: string; body: string; events: RecallEvent[]; question: RecallQuestion }
-export interface RecallFixture { seed: number; disclose: boolean; contract: string; pages: RecallPage[] }
-export interface RecallOptions {
-  stages: number; eventsPerPage: number; paddingChars: number;
-  /**
-   * Whether the contract says later stages may ask about earlier pages.
-   * Default false: disclosed, a diligent model copies every event into a notes
-   * file and the task collapses into F1 (state on disk). Undisclosed is the
-   * case the claim is about -- relevance revealed only after the fact.
-   */
-  disclose?: boolean;
-}
-
-// Log-like filler at roughly 4 chars/token: hex filler tokenizes ~2x denser,
-// which would double the cost of the same nominal length for no realism gain.
-const LEVELS = ["INFO", "INFO", "INFO", "DEBUG", "WARN"];
-const SERVICES = ["api", "worker", "scheduler", "cache", "gateway", "billing", "search", "auth"];
-const MESSAGES = ["heartbeat ok", "request served", "cache refreshed", "retrying upstream call", "job finished",
-  "connection pooled", "config reloaded", "slow query logged", "queue drained", "health check passed"];
-
-const pad = (n: number, width: number) => String(n).padStart(width, "0");
-
-export function recallFixture(seed: number, options: RecallOptions): RecallFixture {
-  const { stages, eventsPerPage, paddingChars } = options, disclose = options.disclose ?? false;
-  if (!Number.isSafeInteger(seed) || !Number.isSafeInteger(stages) || stages < 1 || stages > 256 ||
-      !Number.isSafeInteger(eventsPerPage) || eventsPerPage < 2 || eventsPerPage > 200 ||
-      !Number.isSafeInteger(paddingChars) || paddingChars < 0 || paddingChars > 4_000_000) throw new RangeError("invalid recall fixture dimensions");
-  const random = seededRandom(seed), int = (n: number) => Math.floor(random() * n);
-  const pick = <T>(xs: readonly T[]): T => xs[int(xs.length)]!;
-  const used = new Set<string>();
-  const fresh = (make: () => string) => { for (;;) { const v = make(); if (!used.has(v)) { used.add(v); return v; } } };
-  const hex = (n: number) => Array.from({ length: n }, () => "0123456789abcdef"[int(16)]).join("");
-  // Current state and each ticket's history, replayed exactly as a reader would.
-  const history = new Map<string, { stage: number; code: string }[]>();
-  const asked = new Set<string>();
-  const pages: RecallPage[] = [];
-  for (let stage = 0; stage < stages; stage++) {
-    const earlier = [...history.keys()];
-    // A quarter of events after page 0 reassign an earlier ticket (conflicts).
-    const reassign = stage === 0 ? 0 : Math.min(earlier.length, Math.floor(eventsPerPage / 4));
-    const events: RecallEvent[] = [];
-    const targets = new Set<string>();
-    while (targets.size < reassign) targets.add(pick(earlier));
-    for (const ticket of targets) events.push({ ticket, code: fresh(() => `K-${hex(12)}`), action: "reassign" });
-    while (events.length < eventsPerPage) events.push({ ticket: fresh(() => `T-${pad(10000 + int(90000), 5)}`), code: fresh(() => `K-${hex(12)}`), action: "assign" });
-    // Shuffle so reassignments are not always first on the page.
-    for (let i = events.length - 1; i > 0; i--) { const j = int(i + 1); [events[i], events[j]] = [events[j]!, events[i]!]; }
-    for (const event of events) {
-      const past = history.get(event.ticket) ?? [];
-      past.push({ stage, code: event.code }); history.set(event.ticket, past);
-    }
-
-    const question = chooseQuestion(stage, history, asked, int);
-    asked.add(question.ticket);
-    const lines: string[] = [];
-    for (let usedChars = 0, second = 0; usedChars < paddingChars; second++) {
-      const line = `2026-09-28T${pad(Math.floor(second / 3600) % 24, 2)}:${pad(Math.floor(second / 60) % 60, 2)}:${pad(second % 60, 2)}Z ` +
-        `${pick(LEVELS)} ${pick(SERVICES)}-${int(8)} ${pick(MESSAGES)} latency=${int(900)}ms queue=${int(64)}`;
-      lines.push(line); usedChars += line.length + 1;
-    }
-    // Spread events through the whole page, including its tail, so a head-only
-    // preview cannot pass as a full observation (same rule as F1).
-    for (let i = events.length - 1; i >= 0; i--) {
-      const at = Math.floor((i + 0.5) * lines.length / events.length);
-      lines.splice(at, 0, `EVENT ${JSON.stringify(events[i])}`);
-    }
-    pages.push({ stage, path: `incoming/page-${pad(stage, 2)}.txt`, answerPath: `answers/stage-${pad(stage, 2)}.txt`,
-      body: lines.join("\n"), events, question });
-  }
-  const contract = "This task runs in stages. Each stage delivers one page at incoming/page-NN.txt; the page is deleted when the stage ends. " +
-    "Pages contain EVENT lines (JSON) that assign or reassign a code to a ticket, mixed with unrelated log lines. " +
-    "A ticket's current code is the code from its most recent EVENT across all pages so far. " +
-    (disclose ? "Each stage asks about one ticket, which may have appeared on this page or on any earlier page. "
-      : "Each stage asks about one ticket. ") +
-    "Read the page in full with read, then write only the ticket's current code to the stage's answer file. Use only the provided tools. Reply briefly when the stage is done.";
-  return { seed, disclose, contract, pages };
-}
-
-/** Rotate kinds so every family appears; fall back when a kind has no candidate yet. */
-function chooseQuestion(stage: number, history: Map<string, { stage: number; code: string }[]>, asked: Set<string>, int: (n: number) => number): RecallQuestion {
-  const rows = [...history.entries()].filter(([ticket]) => !asked.has(ticket)).map(([ticket, past]) => ({ ticket, past, last: past.at(-1)! }));
-  const candidates: Record<RecallKind, typeof rows> = {
-    // Asked on the page that last set it, so no retention is needed.
-    recent: rows.filter((row) => row.last.stage === stage),
-    // Set once, on an earlier page: pure retrieval of a detail seen once.
-    recall: rows.filter((row) => row.past.length === 1 && row.last.stage < stage),
-    // Changed on an earlier page: the stale code must be rejected.
-    latest: rows.filter((row) => row.past.length > 1 && row.last.stage < stage),
-  };
-  const rotation: RecallKind[] = ["recent", "recall", "latest"];
-  const preferred = rotation[stage % 3]!;
-  for (const kind of [preferred, "recall", "latest", "recent"] as RecallKind[]) {
-    const pool = candidates[kind];
-    if (pool.length === 0) continue;
-    // Pick the source page first, uniformly, so distances are spread rather
-    // than dominated by whichever page had the most unasked tickets.
-    const sources = [...new Set(pool.map((row) => row.last.stage))].sort((a, b) => a - b);
-    const source = sources[int(sources.length)]!;
-    const onSource = pool.filter((row) => row.last.stage === source);
-    const row = onSource[int(onSource.length)]!;
-    return { stage, kind, ticket: row.ticket, expected: row.last.code, source, stale: row.past.slice(0, -1).map((p) => p.code) };
-  }
-  throw new Error("recall fixture has no question candidate");
-}
-
-export function recallPrompt(fixture: RecallFixture, stage: number): string {
-  const page = fixture.pages[stage]; if (!page) throw new RangeError("unknown recall stage");
-  return (stage === 0 ? fixture.contract + "\n\n" : "Continue under the original rules. ") +
-    `Stage ${stage}: read ${page.path} in full. Question: what is the current code of ticket ${page.question.ticket}? ` +
-    `Write only the code to ${page.answerPath}, then finish this stage.`;
-}
-
-export type RecallVerdict = { ok: true; reason: "exact" } | { ok: false; reason: "missing" | "stale" | "other_ticket" | "wrong" };
-
-/** Only whitespace and one wrapping quote/backtick pair are forgiven. */
-export function verifyRecallAnswer(body: string | null, question: RecallQuestion, codes: ReadonlySet<string>): RecallVerdict {
-  if (body === null) return { ok: false, reason: "missing" };
-  let answer = body.trim();
-  const wrapped = /^(["'`])(.*)\1$/s.exec(answer);
-  if (wrapped) answer = wrapped[2]!.trim();
-  if (answer === "") return { ok: false, reason: "missing" };
-  if (answer === question.expected) return { ok: true, reason: "exact" };
-  if (question.stale.includes(answer)) return { ok: false, reason: "stale" };
-  return { ok: false, reason: codes.has(answer) ? "other_ticket" : "wrong" };
-}
-
-/** Every code any page issued, to tell a confused lookup from a fabrication. */
-export function recallCodes(fixture: RecallFixture): Set<string> {
-  return new Set(fixture.pages.flatMap((page) => page.events.map((event) => event.code)));
-}