@visual-z/xeno 0.7.1 → 0.8.0

package/src/core/experiment.ts

@@ -1,173 +0,0 @@
-/** Experimental projections never mutate the durable event stream (D8, L). */
-import type { Message } from "./project.ts";
-import { isTrialLifecycle, type TrialLifecycle } from "./trial-lifecycle.ts";
-import { RECOVERY_TASKS, recoveryCase, type RecoveryTask } from "./recovery-corpus.ts";
-import { isRecoveryMetrics, type RecoveryMetrics } from "./recovery-evaluation.ts";
-import { TOOL_MODE_FACTORS, TOOL_MODE_TASKS, isToolModeTask, isToolModeMetrics, type ToolModeTask, type ToolModeMetrics } from "./tool-mode-protocol.ts";
-
-export type ContextArm = "node" | "replay" | "truncate" | "mask";
-
-export function experimentalContext(messages: readonly Message[], arm: ContextArm, keepRounds: number): Message[] {
-  if (!Number.isSafeInteger(keepRounds) || keepRounds < 1) throw new RangeError("keepRounds must be positive");
-  if (arm === "node" || arm === "replay") return [...messages];
-  const groups: Message[][] = [[]];
-  for (const message of messages) {
-    if (message.role === "assistant") groups.push([]);
-    groups.at(-1)!.push(message);
-  }
-  const recent = Math.max(1, groups.length - keepRounds);
-  if (arm === "truncate") return [...groups[0]!, ...groups.slice(recent).flat()];
-  return groups.flatMap((group, i) => i >= recent ? group : group.map((message) => ({
-    ...message,
-    content: message.content.map((block) => block.type === "tool_result"
-      ? { ...block, content: "[observation masked by experiment; original remains in log]" } : block),
-  })));
-}
-
-/** Fisher-Yates with a fixed seed; scheduling and intervals are reproducible. */
-export function seededRandom(seed: number): () => number {
-  let state = seed >>> 0;
-  return () => {
-    state = (Math.imul(1664525, state) + 1013904223) >>> 0;
-    return state / 4294967296;
-  };
-}
-
-export function shuffled<T>(items: readonly T[], seed: number): T[] {
-  const random = seededRandom(seed);
-  const out = [...items];
-  for (let i = out.length - 1; i > 0; i--) {
-    const j = Math.floor(random() * (i + 1));
-    [out[i], out[j]] = [out[j]!, out[i]!];
-  }
-  return out;
-}
-
-/** O.16: each contrast changes exactly one policy, with paired task repeats. */
-export const EVAL_FACTORS = ["profile", "workingSet", "strictVerification"] as const;
-export const EVAL_TASKS = ["short", "boundary-recall", "false-completion", "human-boundary"] as const;
-const supportedFactors = [...EVAL_FACTORS, ...TOOL_MODE_FACTORS] as const;
-export type EvalFactor = typeof supportedFactors[number];
-export type EvalTask = typeof EVAL_TASKS[number] | RecoveryTask | ToolModeTask;
-const supportedTasks:readonly EvalTask[]=[...EVAL_TASKS,...RECOVERY_TASKS,...TOOL_MODE_TASKS];
-export type EvalArm = "control" | "treatment";
-export type EvalPolicy = { profile: "full" | "coding"; workingSet: boolean; strictVerification: boolean; querySyntax?: "json" | "sql" };
-export type EvalJob = { id: string; factor: EvalFactor; task: EvalTask; repeat: number; arm: EvalArm; policy: EvalPolicy };
-export function evalPolicy(factor: EvalFactor, arm: EvalArm): EvalPolicy {
-  const policy: EvalPolicy = { profile: "coding", workingSet: true, strictVerification: true };
-  if (factor === "querySyntax") return { ...policy, querySyntax: arm === "treatment" ? "sql" : "json" };
-  if (arm === "control") {
-    if (factor === "profile") policy.profile = "full";
-    else policy[factor] = false;
-  }
-  return policy;
-}
-export function singleFactorSchedule(seed: number, repeats = 1, tasks: readonly EvalTask[] = EVAL_TASKS, factors: readonly EvalFactor[] = EVAL_FACTORS): EvalJob[] {
-  if (!Number.isSafeInteger(seed) || !Number.isSafeInteger(repeats) || repeats < 1 || repeats > 20 || !tasks.length ||
-      new Set(tasks).size !== tasks.length || tasks.some(task => !supportedTasks.includes(task)) || !factors.length || new Set(factors).size !== factors.length ||
-      factors.some(factor => !supportedFactors.includes(factor) || tasks.some(task => isToolModeTask(task) !== (TOOL_MODE_FACTORS as readonly string[]).includes(factor)))) throw new RangeError("invalid eval schedule");
-  const pairs = shuffled(factors.flatMap(factor => [...tasks].sort().flatMap(task =>
-    Array.from({ length: repeats }, (_, repeat) => ({ factor, task, repeat })))), seed);
-  // Alternate first position before shuffling: adjacent LCG seeds are correlated.
-  const positions = new Map<EvalFactor, number>();
-  const order = shuffled(["control", "treatment"] as const, seed + 1);
-  return pairs.flatMap(pair => {
-    const offset = positions.get(pair.factor) ?? 0; positions.set(pair.factor, offset + 1);
-    return order.map((_, i) => {
-      const arm = order[(offset + i) % 2]!;
-      return { ...pair, arm, id: `${pair.factor}/${pair.task}/${pair.repeat}/${arm}`, policy: evalPolicy(pair.factor, arm) };
-    });
-  });
-}
-export function validateEvalJobs(jobs: readonly EvalJob[]): void {
-  const seen = new Set<string>();
-  for (const job of jobs) {
-    if (!supportedFactors.includes(job.factor) || !supportedTasks.includes(job.task) || !["control", "treatment"].includes(job.arm) ||
-        isToolModeTask(job.task) !== (TOOL_MODE_FACTORS as readonly string[]).includes(job.factor) ||
-        !Number.isSafeInteger(job.repeat) || job.repeat < 0 || job.id !== `${job.factor}/${job.task}/${job.repeat}/${job.arm}` || seen.has(job.id) ||
-        JSON.stringify(job.policy) !== JSON.stringify(evalPolicy(job.factor, job.arm))) throw new Error("invalid or confounded eval job");
-    seen.add(job.id);
-  }
-  for (const job of jobs) if (!seen.has(`${job.factor}/${job.task}/${job.repeat}/${job.arm === "control" ? "treatment" : "control"}`)) throw new Error("unpaired eval job");
-}
-export interface EvalResult {
-  toolMode?: ToolModeMetrics;
-  recovery?: RecoveryMetrics;
-  /** v2 separates an unmet final state from a machine-readable unsupported claim. */
-  unverifiedEnding?: boolean;
-  /** Absent in the frozen O.16 archives; absence is unknown, not an uninterrupted run. */
-  lifecycle?: TrialLifecycle;
-  id: string;
-  status: "finished" | "error" | "budget" | "stopped";
-  scenarioPassed: boolean;
-  artifactCorrect: boolean;
-  verified: boolean;
-  awaitingHuman: boolean;
-  falseCompletion: boolean;
-  requests: number;
-  toolFailures: number;
-  repeatedReads: number;
-  promptTokens: number | null;
-  cacheTokens: number | null;
-  outputTokens: number | null;
-  costUsd: number | null;
-  reservedUnknownUsd: number;
-}
-export function summarizeSingleFactor(jobs: readonly EvalJob[], results: readonly EvalResult[]) {
-  validateEvalJobs(jobs);
-  const byId = new Map<string, EvalResult>(), planned = new Set(jobs.map(job => job.id));
-  for (const row of results) {
-    if (row.toolMode !== undefined && (!isToolModeMetrics(row.toolMode) || row.scenarioPassed && !row.toolMode.sourceComplete)) throw Error("invalid tool mode metrics");
-    if (row.unverifiedEnding !== undefined && typeof row.unverifiedEnding !== "boolean") throw Error("invalid ending metric");
-    if(row.recovery!==undefined&&(!isRecoveryMetrics(row.recovery)||row.recovery.finalTaskSuccess!==row.scenarioPassed||row.recovery.toolFailures!==row.toolFailures||row.falseCompletion!==(row.recovery.unsupportedClaims>0)))throw Error("invalid recovery metrics");
-    if (row.lifecycle !== undefined && (!isTrialLifecycle(row.lifecycle) || row.lifecycle.backendRequests > row.requests ||
-        row.lifecycle.interruption !== null && row.status === "finished" || row.lifecycle.cleanup === "failed" && row.status !== "error"))
-      throw new Error("invalid eval lifecycle result");
-    if (!planned.has(row.id) || byId.has(row.id) || !["finished", "error", "budget", "stopped"].includes(row.status) ||
-        [row.scenarioPassed, row.artifactCorrect, row.verified, row.awaitingHuman, row.falseCompletion].some(value => typeof value !== "boolean") ||
-        row.scenarioPassed && (row.status !== "finished" || !row.verified || row.falseCompletion && row.recovery === undefined) ||
-        [row.requests, row.toolFailures, row.repeatedReads].some(n => !Number.isSafeInteger(n) || n < 0) ||
-        [row.promptTokens, row.cacheTokens, row.outputTokens].some(n => n !== null && (!Number.isSafeInteger(n) || n < 0)) ||
-        [row.costUsd, row.reservedUnknownUsd].some(n => n !== null && (!Number.isFinite(n) || n < 0)) ||
-        row.cacheTokens !== null && row.promptTokens !== null && row.cacheTokens > row.promptTokens) throw new Error("invalid or duplicate eval result");
-    byId.set(row.id, row);
-  }
-  const factors = supportedFactors.filter(factor => jobs.some(job => job.factor === factor));
-  const arms = factors.flatMap(factor => (["control", "treatment"] as const).map(arm => {
-    const selected = jobs.filter(job => job.factor === factor && job.arm === arm), rows = selected.flatMap(job => byId.has(job.id) ? [byId.get(job.id)!] : []);
-    const complete = rows.length === selected.length, scenarioPassed = rows.filter(row => row.scenarioPassed).length;
-    const knownCostSubtotal = rows.reduce((sum, row) => sum + (row.costUsd ?? 0), 0);
-    // Missing jobs and unknown charges cannot become a zero-priced success.
-    const costUsd = complete && rows.length > 0 && rows.every(row => row.costUsd !== null) ? knownCostSubtotal : null;
-    const total = (field: "promptTokens" | "cacheTokens" | "outputTokens") => complete && rows.length > 0 && rows.every(row => row[field] !== null)
-      ? rows.reduce((sum, row) => sum + row[field]!, 0) : null;
-    const recovery=selected.some(job=>recoveryCase(job.task)||isToolModeTask(job.task))?{observed:rows.filter(row=>row.recovery).length,
-      missing:selected.length-rows.filter(row=>row.recovery).length,
-      firstToolSuccess:rows.filter(row=>row.recovery?.firstToolSuccess===true).length,
-      firstPassTaskSuccess:rows.filter(row=>row.recovery?.firstPassTaskSuccess===true).length,
-      ...Object.fromEntries((["toolCalls","outerCalls","nestedCalls","unresolvedCalls","recoveryCalls","unchangedFailureRetries","originalEvidenceRetrievals","unsupportedClaims"] as const)
-        .map(key=>[key,rows.reduce((n,row)=>n+(row.recovery?.[key]??0),0)]))}:undefined;
-    return { factor, arm, ...(recovery?{recovery}:{}), planned: selected.length, attempted: rows.length, missing: selected.length - rows.length,
-      failed: rows.filter(row => !row.scenarioPassed).length, scenarioPassed, verified: rows.filter(row => row.verified).length,
-      awaitingHuman: rows.filter(row => row.awaitingHuman).length, falseCompletions: rows.filter(row => row.falseCompletion).length,
-      plannedPassFraction: selected.length ? scenarioPassed / selected.length : null, complete,
-      requests: rows.reduce((n, row) => n + row.requests, 0), toolFailures: rows.reduce((n, row) => n + row.toolFailures, 0),
-      repeatedReads: rows.reduce((n, row) => n + row.repeatedReads, 0), promptTokens: total("promptTokens"), cacheTokens: total("cacheTokens"), outputTokens: total("outputTokens"),
-      knownCostSubtotal, costUsd, unknownCostJobs: rows.filter(row => row.costUsd === null).length,
-      reservedUnknownUsd: rows.reduce((n, row) => n + row.reservedUnknownUsd, 0),
-      costPerScenarioSuccess: costUsd !== null && scenarioPassed > 0 ? costUsd / scenarioPassed : null };
-  }));
-  const comparisons = factors.map(factor => {
-    const pairs = jobs.filter(job => job.factor === factor && job.arm === "control");
-    const clusters = [...new Set(pairs.map(job => job.task))].sort().map(task => {
-      const matched = pairs.filter(job => job.task === task).flatMap(job => {
-        const control = byId.get(job.id), treatment = byId.get(`${factor}/${task}/${job.repeat}/treatment`);
-        return control && treatment ? [{ delta: Number(treatment.scenarioPassed) - Number(control.scenarioPassed) }] : [];
-      });
-      return { task, matchedPairs: matched.length, passDelta: matched.length ? matched.reduce((n, pair) => n + pair.delta, 0) / matched.length : null };
-    });
-    const matchedPairs = clusters.reduce((n, row) => n + row.matchedPairs, 0);
-    return { factor, matchedPairs, missingPairs: pairs.length - matchedPairs, clusters };
-  });
-  return { arms, comparisons };
-}

package/src/core/harness-comparison.ts

@@ -1,90 +0,0 @@
-/** Shared evaluation protocol, not a production context policy (research N.10). */
-import { shuffled } from "./experiment.ts";
-import { canReserve, type BenchmarkUsage } from "./benchmark.ts";
-
-export const COMPARISON_ARMS = ["xeno-windowed", "xeno-continuous", "pi"] as const;
-export type ComparisonArm = typeof COMPARISON_ARMS[number];
-export interface ComparisonJob { id: string; arm: ComparisonArm }
-
-/** Null means an explicit provider-balance authorization, never zero allowance. */
-export function comparisonCanReserve(charged: number, reservation: number, budget: number | null): boolean {
-  if (![charged, reservation, charged + reservation].every(Number.isFinite) || charged < 0 || reservation <= 0) return false;
-  return budget === null || canReserve(charged, reservation, budget);
-}
-
-/** DeepSeek documents 402 as insufficient balance; other errors need their own diagnosis. */
-export function comparisonFailureKind(status: number | null): "balance" | "provider" {
-  return status === 402 ? "balance" : "provider";
-}
-
-export function comparisonSchedule(ids: readonly string[], seed: number): ComparisonJob[] {
-  if (ids.length === 0 || new Set(ids).size !== ids.length) throw new Error("empty or duplicate comparison tasks");
-  const arms = shuffled(COMPARISON_ARMS, seed);
-  return [...ids].sort().flatMap((id, i) => arms.map((_, position) => ({ id, arm: arms[(i + position) % arms.length]! })));
-}
-
-export function jsonObject(value: unknown): Record<string, unknown> {
-  if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("expected JSON object");
-  return value as Record<string, unknown>;
-}
-
-/** Missing cache usage is unknown, never a fabricated zero or a discounted bill. */
-export function comparisonUsage(body: unknown): BenchmarkUsage | null {
-  try {
-    const usage = jsonObject(jsonObject(body)["usage"]);
-    const prompt = usage["prompt_tokens"], output = usage["completion_tokens"];
-    const details = usage["prompt_tokens_details"];
-    const vendorCached = usage["prompt_cache_hit_tokens"];
-    const standardCached = details ? jsonObject(details)["cached_tokens"] : undefined;
-    if (vendorCached !== undefined && standardCached !== undefined && vendorCached !== standardCached) return null;
-    const cached = vendorCached ?? standardCached;
-    if (![prompt, output, cached].every((n) => typeof n === "number" && Number.isSafeInteger(n) && n >= 0)) return null;
-    const p = prompt as number, o = output as number, c = cached as number;
-    if (p === 0 || o === 0 || c > p) return null;
-    if (usage["prompt_cache_miss_tokens"] !== undefined && usage["prompt_cache_miss_tokens"] !== p - c) return null;
-    return { input_tokens: p - c, output_tokens: o, cache_read_input_tokens: c, cache_creation_input_tokens: 0 };
-  } catch { return null; }
-}
-
-/** Buffer upstream responses for one billing boundary; do not report this as TTFT. */
-export function completionSse(value: unknown): string {
-  const body = jsonObject(value);
-  const choices = body["choices"];
-  if (!Array.isArray(choices) || choices.length !== 1) throw new Error("expected one completion");
-  const choice = jsonObject(choices[0]), message = jsonObject(choice["message"]);
-  const calls = message["tool_calls"];
-  const finish = choice["finish_reason"];
-  if (typeof finish !== "string" || !["stop", "tool_calls", "length", "content_filter"].includes(finish)) throw new Error("incomplete completion");
-  if (calls !== undefined && !Array.isArray(calls)) throw new Error("invalid tool calls");
-  const delta = { ...message, ...(Array.isArray(calls) ? { tool_calls: calls.map((call: unknown, index: number) => ({ ...jsonObject(call), index })) } : {}) };
-  const common = { id: body["id"], object: "chat.completion.chunk", created: body["created"], model: body["model"] };
-  return [
-    { ...common, choices: [{ index: 0, delta, finish_reason: null }] },
-    { ...common, choices: [{ index: 0, delta: {}, finish_reason: finish }] },
-    { ...common, choices: [], usage: body["usage"] },
-  ].map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n";
-}
-
-/** JSONL permits U+2028 in strings. Only LF frames an event, including Pi's output. */
-export function comparisonLines(buffer: string, chunk: string): { buffer: string; values: unknown[] } {
-  const lines = (buffer + chunk).split("\n");
-  const remainder = lines.pop()!;
-  return { buffer: remainder, values: lines.filter((line) => line.trim() !== "").map((line) => JSON.parse(line) as unknown) };
-}
-
-export function comparisonSummary(plan: readonly ComparisonJob[], results: readonly (ComparisonJob & { status: string; correct: boolean })[]) {
-  const keys = new Set(plan.map((job) => `${job.id}/${job.arm}`)), seen = new Set<string>();
-  if (keys.size !== plan.length) throw new Error("duplicate planned comparison job");
-  for (const result of results) {
-    const key = `${result.id}/${result.arm}`;
-    if (!keys.has(key) || seen.has(key) || (result.correct && result.status !== "answered")) throw new Error("invalid comparison outcome");
-    seen.add(key);
-  }
-  return COMPARISON_ARMS.map((arm) => {
-    const planned = plan.filter((job) => job.arm === arm).length, rows = results.filter((row) => row.arm === arm);
-    const correct = rows.filter((row) => row.correct).length;
-    return { arm, planned, attempted: rows.length, answered: rows.filter((row) => row.status === "answered").length,
-      correct, missing: planned - rows.length, accuracy: rows.length ? correct / rows.length : null,
-      complete: rows.length === planned };
-  }).filter((row) => row.planned > 0);
-}

package/src/core/harness-diagnostics.ts

@@ -1,206 +0,0 @@
-/** D11 / O.3: classify durable execution facts, never error words in tool output. */
-import type { LogEvent, ToolResultEvent } from "./events.ts";
-import type { BenchmarkUsage } from "./benchmark.ts";
-import { canonicalValue } from "./continuation.ts";
-
-export type ResultClass = "success" | "invalid_input" | "cancelled" | "timeout" | "command_nonzero" |
-  "execution_failed" | "rejected" | "unclassified_failure";
-
-export function classifyToolResult(event: ToolResultEvent): ResultClass {
-  const code = event.diagnostic?.code;
-  if (code === "invalid_input") return "invalid_input";
-  if (code === "cancelled" || event.outcome?.reason === "cancelled") return "cancelled";
-  if (code === "timeout" || event.outcome?.reason === "timeout") return "timeout";
-  if (event.effect?.kind === "command" && event.effect.exit !== 0) return "command_nonzero";
-  if (event.ok) return "success";
-  if (event.diagnostic?.execution === "not_started" ||
-    ["version_mismatch", "not_found", "ambiguous_edit", "permission_denied", "environment_missing"].includes(code ?? "")) return "rejected";
-  if (code === "execution_failed" || code === "unknown_effect" || event.outcome?.reason === "execution_error") return "execution_failed";
-  return "unclassified_failure";
-}
-
-export interface DiagnosticRequest {
-  request: number;
-  node: number | null;
-  usage: BenchmarkUsage | null;
-  responseModel: string | null;
-  failure: string | null;
-  firstDeltaMs: number | null;
-  wallMs: number | null;
-  prefixUnchanged: boolean | null;
-  sharedInputBytes: number | null;
-  frozenSha256: string | null;
-  /** Raw terminal counts can remain known when a cache breakdown is absent. */
-  reportedPromptTokens?: number | null;
-  reportedOutputTokens?: number | null;
-}
-export interface ToolCounters {
-  calls: number; providerCalls: number; invocations: number; nestedInvocations: number;
-  results: number; pairedCalls: number; orphanCalls: number; orphanResults: number;
-  duplicateProviderCalls: number; duplicateInvocations: number; duplicateResults: number;
-  repeatedProviderInputs: number; repeatedInvocationInputs: number;
-  classes: Record<ResultClass, number>;
-  diagnosticCodes: Record<string, number>;
-  commandExits: Record<string, number>;
-  expectedNonzeroCommands: number;
-  undeclaredNonzeroCommands: number;
-}
-const classes = (): Record<ResultClass, number> => ({ success: 0, invalid_input: 0, cancelled: 0, timeout: 0,
-  command_nonzero: 0, execution_failed: 0, rejected: 0, unclassified_failure: 0 });
-const counters = (): ToolCounters => ({ calls: 0, providerCalls: 0, invocations: 0, nestedInvocations: 0,
-  results: 0, pairedCalls: 0, orphanCalls: 0, orphanResults: 0, duplicateProviderCalls: 0,
-  duplicateInvocations: 0, duplicateResults: 0, repeatedProviderInputs: 0, repeatedInvocationInputs: 0,
-  classes: classes(), diagnosticCodes: {}, commandExits: {}, expectedNonzeroCommands: 0, undeclaredNonzeroCommands: 0 });
-function add(record: Record<string, number>, key: string): void { record[key] = (record[key] ?? 0) + 1; }
-const validCount = (value: number): boolean => Number.isSafeInteger(value) && value >= 0;
-const reportedCount = (value: number | null | undefined): number | null => typeof value === "number" && validCount(value) ? value : null;
-const validTime = (value: number | null): number | null => value !== null && Number.isFinite(value) && value >= 0 ? value : null;
-function observedUsage(usage: BenchmarkUsage | null): BenchmarkUsage | null {
-  return usage && [usage.input_tokens, usage.output_tokens, usage.cache_read_input_tokens,
-    usage.cache_creation_input_tokens].every(validCount) ? usage : null;
-}
-function quantile(values: readonly number[], q: number): number | null {
-  return values.length ? values.toSorted((a, b) => a - b)[Math.max(0, Math.ceil(values.length * q) - 1)]! : null;
-}
-interface Call {
-  name: string; provider: boolean; invocation: boolean; nested: boolean;
-  providerInput: string | null; invocationInput: string | null;
-}
-
-/**
- * Inputs and outputs remain private. Equal-input counts use canonical original
- * model arguments and immutable admitted-input hashes separately (O.3 / Pi A).
- * Expected nonzero exits need an evaluator declaration; an exit alone cannot
- * prove a command was accidental, and model prose cannot establish intent.
- */
-export function summarizeHarnessDiagnostics(input: {
-  events: readonly LogEvent[]; requests: readonly DiagnosticRequest[];
-  expectedNonzeroCallIds?: readonly string[];
-}) {
-  const tools: Record<string, ToolCounters> = {};
-  const tool = (name: string): ToolCounters => tools[name] ??= counters();
-  const calls = new Map<string, Call>(), results = new Map<string, ToolResultEvent[]>(), seenEvents = new Set<string>();
-  const guidance: Record<string, number> = {}, cuts: Record<string, number> = {}, cleanupProcessExits: Record<string, number> = {};
-  let duplicateEvents = 0, interruptions = 0;
-  for (const event of input.events) {
-    if (seenEvents.has(event.id)) { duplicateEvents++; continue; }
-    seenEvents.add(event.id);
-    if (event.t === "guidance") add(guidance, event.reason);
-    if (event.t === "cut") add(cuts, event.reason);
-    if (event.t === "interrupt") interruptions++;
-    if (event.t === "process_exit") add(cleanupProcessExits, String(event.exit));
-    if (event.t === "assistant") for (const block of event.blocks) {
-      if (block.type !== "tool_use") continue;
-      const existing = calls.get(block.call_id), stats = tool(block.name);
-      if (existing?.provider) stats.duplicateProviderCalls++;
-      calls.set(block.call_id, { name: existing?.name ?? block.name, provider: true,
-        invocation: existing?.invocation ?? false, nested: existing?.nested ?? false,
-        providerInput: canonicalValue(block.input), invocationInput: existing?.invocationInput ?? null });
-    }
-    if (event.t === "tool_invocation") {
-      const existing = calls.get(event.call_id), stats = tool(event.name);
-      if (existing?.invocation) stats.duplicateInvocations++;
-      calls.set(event.call_id, { name: existing?.name ?? event.name, provider: existing?.provider ?? false,
-        invocation: true, nested: event.parent_call_id !== undefined,
-        providerInput: existing?.providerInput ?? null, invocationInput: event.input_ref.sha256 });
-    }
-    if (event.t === "tool_result") {
-      const found = results.get(event.call_id) ?? [];
-      found.push(event); results.set(event.call_id, found);
-    }
-  }
-  const providerInputs = new Map<string, Set<string>>(), invocationInputs = new Map<string, Set<string>>();
-  const repeat = (groups: Map<string, Set<string>>, name: string, key: string | null): boolean => {
-    if (key === null) return false;
-    const seen = groups.get(name) ?? new Set<string>(), repeated = seen.has(key);
-    seen.add(key); groups.set(name, seen); return repeated;
-  };
-  for (const [id, call] of calls) {
-    const stats = tool(call.name); stats.calls++;
-    if (call.provider) stats.providerCalls++;
-    if (call.invocation) stats.invocations++;
-    if (call.nested) stats.nestedInvocations++;
-    if (repeat(providerInputs, call.name, call.providerInput)) stats.repeatedProviderInputs++;
-    if (repeat(invocationInputs, call.name, call.invocationInput)) stats.repeatedInvocationInputs++;
-    if (results.has(id)) stats.pairedCalls++; else stats.orphanCalls++;
-  }
-  const expected = new Set(input.expectedNonzeroCallIds ?? []);
-  for (const [id, found] of results) {
-    const call = calls.get(id), stats = tool(call?.name ?? "<orphan>");
-    stats.duplicateResults += Math.max(0, found.length - 1);
-    for (const event of found) {
-      stats.results++;
-      if (!call) stats.orphanResults++;
-      stats.classes[classifyToolResult(event)]++;
-      if (event.diagnostic) add(stats.diagnosticCodes, event.diagnostic.code);
-      if (event.effect?.kind === "command") {
-        add(stats.commandExits, String(event.effect.exit));
-        if (event.effect.exit !== 0) {
-          if (expected.has(id)) stats.expectedNonzeroCommands++; else stats.undeclaredNonzeroCommands++;
-        }
-      }
-    }
-  }
-  const requestIds = new Set<number>(), requests: {
-    request: number; node: number | null; usageKnown: boolean; promptTokens: number | null;
-    cachedTokens: number | null; cacheWriteTokens: number | null; outputTokens: number | null;
-    cacheHitRate: number | null; responseModel: string | null; failure: string | null;
-    firstDeltaMs: number | null; wallMs: number | null; prefixUnchanged: boolean | null; sharedInputBytes: number | null;
-  }[] = [];
-  const models: Record<string, number> = {};
-  const usage = { knownRequests: 0, unknownRequests: 0, requestsWithCacheReads: 0, promptTokens: 0, cachedTokens: 0,
-    cacheWriteTokens: 0, outputTokens: 0, promptKnownRequests: 0, promptUnknownRequests: 0,
-    cacheCoveredPromptTokens: 0, cacheCoverageRate: null as number | null,
-    observedCacheHitRate: null as number | null, cacheHitRate: null as number | null };
-  const prefix = { sameNodeComparisons: 0, sameNodeMutations: 0, boundaryComparisons: 0, boundaryMutations: 0,
-    unknownComparisons: 0, sameNodeFrozenMutations: 0, boundaryFrozenMutations: 0 };
-  let previous: DiagnosticRequest | undefined, duplicateRequests = 0;
-  for (const request of input.requests.toSorted((a, b) => a.request - b.request)) {
-    if (requestIds.has(request.request)) { duplicateRequests++; continue; }
-    requestIds.add(request.request);
-    const observed = observedUsage(request.usage);
-    const prompt = reportedCount(request.reportedPromptTokens) ?? (observed ?
-      observed.input_tokens + observed.cache_read_input_tokens + observed.cache_creation_input_tokens : null);
-    const output = reportedCount(request.reportedOutputTokens) ?? observed?.output_tokens ?? null;
-    if (prompt !== null) { usage.promptKnownRequests++; usage.promptTokens += prompt; } else usage.promptUnknownRequests++;
-    if (output !== null) usage.outputTokens += output;
-    if (observed) {
-      usage.knownRequests++; usage.cacheCoveredPromptTokens += prompt!; usage.cachedTokens += observed.cache_read_input_tokens;
-      usage.cacheWriteTokens += observed.cache_creation_input_tokens;
-      if (observed.cache_read_input_tokens > 0) usage.requestsWithCacheReads++;
-    } else usage.unknownRequests++;
-    if (request.responseModel !== null) add(models, request.responseModel);
-    if (previous) {
-      if (previous.node === null || request.node === null || request.prefixUnchanged === null) prefix.unknownComparisons++;
-      else if (previous.node === request.node) { prefix.sameNodeComparisons++; if (!request.prefixUnchanged) prefix.sameNodeMutations++; }
-      else { prefix.boundaryComparisons++; if (!request.prefixUnchanged) prefix.boundaryMutations++; }
-      if (previous.frozenSha256 !== null && request.frozenSha256 !== null && previous.frozenSha256 !== request.frozenSha256) {
-        if (previous.node === request.node && request.node !== null) prefix.sameNodeFrozenMutations++;
-        else if (previous.node !== null && request.node !== null) prefix.boundaryFrozenMutations++;
-      }
-    }
-    previous = request;
-    requests.push({ request: request.request, node: request.node, usageKnown: observed !== null, promptTokens: prompt,
-      cachedTokens: observed?.cache_read_input_tokens ?? null, cacheWriteTokens: observed?.cache_creation_input_tokens ?? null,
-      outputTokens: output, cacheHitRate: observed && prompt ? observed.cache_read_input_tokens / prompt : null,
-      responseModel: request.responseModel, failure: request.failure, firstDeltaMs: validTime(request.firstDeltaMs),
-      wallMs: validTime(request.wallMs), prefixUnchanged: request.prefixUnchanged, sharedInputBytes: request.sharedInputBytes });
-  }
-  usage.observedCacheHitRate = usage.cacheCoveredPromptTokens > 0 ? usage.cachedTokens / usage.cacheCoveredPromptTokens : null;
-  usage.cacheCoverageRate = usage.promptUnknownRequests === 0 && usage.promptTokens > 0 ? usage.cacheCoveredPromptTokens / usage.promptTokens : null;
-  usage.cacheHitRate = usage.unknownRequests === 0 && usage.promptTokens > 0 ? usage.cachedTokens / usage.promptTokens : null;
-  const delta = requests.flatMap(r => r.firstDeltaMs === null ? [] : [r.firstDeltaMs]);
-  const totals = counters();
-  for (const stats of Object.values(tools)) {
-    for (const key of Object.keys(totals) as (keyof ToolCounters)[]) {
-      const value = stats[key];
-      if (typeof value === "number") (totals[key] as number) += value;
-    }
-    for (const key of Object.keys(stats.classes) as ResultClass[]) totals.classes[key] += stats.classes[key];
-    for (const key of Object.keys(stats.commandExits)) totals.commandExits[key] = (totals.commandExits[key] ?? 0) + stats.commandExits[key]!;
-    for (const key of Object.keys(stats.diagnosticCodes)) totals.diagnosticCodes[key] = (totals.diagnosticCodes[key] ?? 0) + stats.diagnosticCodes[key]!;
-  }
-  return { eventCount: seenEvents.size, duplicateEvents, tools, totals, guidance, cuts, interruptions, cleanupProcessExits,
-    requestCount: requests.length, duplicateRequests, models, usage, prefix, requests,
-    firstDeltaMs: { samples: delta.length, missing: requests.length - delta.length, p50: quantile(delta, 0.5), p95: quantile(delta, 0.95) } };
-}

package/src/core/native-eval.ts

@@ -1,25 +0,0 @@
-/** Q.20: native subscription measurements are standard API price equivalents, never invoices. */
-import { measuredCharge, requestReservation, type BenchmarkPrice, type BenchmarkUsage } from "./benchmark.ts";
-
-// Verified Luna short-context tier. Bound wire bytes conservatively before dispatch;
-// crossing this tier needs a new protocol, not silent use of the cheaper price.
-export const NATIVE_EVAL_INPUT_LIMIT = 272000;
-export function nativeUsage(response: unknown): BenchmarkUsage | null {
-  if (!response || typeof response !== "object") return null;
-  const raw = (response as Record<string, unknown>)["usage"];
-  if (!raw || typeof raw !== "object") return null;
-  const u = raw as Record<string, unknown>, details = u["input_tokens_details"] as Record<string, unknown> | undefined;
-  const input = u["input_tokens"], output = u["output_tokens"], cached = details?.["cached_tokens"] ?? 0, created = details?.["cache_write_tokens"] ?? 0;
-  if (![input, output, cached, created].every(n => typeof n === "number" && Number.isSafeInteger(n) && n >= 0) ||
-      Number(input) <= 0 || Number(output) <= 0 || Number(input) > NATIVE_EVAL_INPUT_LIMIT || Number(cached) + Number(created) > Number(input)) return null;
-  return { input_tokens: Number(input) - Number(cached) - Number(created), output_tokens: Number(output),
-    cache_read_input_tokens: Number(cached), cache_creation_input_tokens: Number(created) };
-}
-export function nativeReservation(bytes: number, modelMaxOutput: number, price: BenchmarkPrice): number {
-  if (bytes + 4096 > NATIVE_EVAL_INPUT_LIMIT || price.cacheWrite === undefined) throw Error("native eval price tier or cache-write rate is not covered");
-  return requestReservation(bytes, modelMaxOutput, { ...price, input: Math.max(price.input, price.cacheWrite) });
-}
-export function nativeMeasured(response: unknown, price: BenchmarkPrice): number | null {
-  const usage = nativeUsage(response);
-  return usage && price.cacheWrite !== undefined ? measuredCharge(usage, price) : null;
-}

package/src/core/policy-evaluation.ts

@@ -1,55 +0,0 @@
-/** N.7: repetitions stay attached to their task; they are not independent questions. */
-import { type ContextPolicy } from "./context-policy.ts";
-import { shuffled } from "./experiment.ts";
-
-// N.7 is a frozen three-arm protocol; new production policies cannot change it.
-const EVALUATED_POLICIES: readonly ContextPolicy[] = ["segmented", "continuous", "adaptive"];
-
-export interface PolicyJob { id: string; policy: ContextPolicy; repeat: number }
-export function policyJobKey(job: PolicyJob): string { return JSON.stringify([job.id, job.policy, job.repeat]); }
-export function policySchedule(ids: readonly string[], repetitions: number, seed: number): PolicyJob[] {
-  if (ids.length === 0 || new Set(ids).size !== ids.length || !Number.isSafeInteger(repetitions) || repetitions < 1) throw new Error("invalid policy trial design");
-  const policies = shuffled(EVALUATED_POLICIES, seed);
-  const jobs: PolicyJob[] = [];
-  let block = 0;
-  for (let repeat = 0; repeat < repetitions; repeat++) for (const id of shuffled([...ids].sort(), seed + repeat + 1)) {
-    for (let position = 0; position < policies.length; position++) jobs.push({ id, repeat, policy: policies[(block + position) % policies.length]! });
-    block++;
-  }
-  return jobs;
-}
-export interface PolicyObservation extends PolicyJob {
-  status: string; correct: boolean; peakRateUsd: number; reservedUnknownUsd: number;
-  requests: number; cuts: number; incompleteHandoffs: number; wallMs: number;
-  usage: { prompt_tokens: number; cache_read_input_tokens: number; output_tokens: number };
-}
-export function summarizePolicies(plan: readonly PolicyJob[], results: readonly PolicyObservation[]) {
-  const planned = new Set(plan.map(policyJobKey)), observed = new Set<string>();
-  if (planned.size !== plan.length) throw new Error("duplicate planned policy job");
-  for (const result of results) {
-    const key = policyJobKey(result);
-    if (!planned.has(key) || observed.has(key) || (result.correct && result.status !== "answered")) throw new Error("invalid policy result");
-    observed.add(key);
-  }
-  return [...new Set(plan.map((job) => job.policy))].sort().map((policy) => {
-    const schedule = plan.filter((job) => job.policy === policy), group = results.filter((row) => row.policy === policy);
-    const sum = (read: (row: PolicyObservation) => number) => group.reduce((n, row) => n + read(row), 0);
-    const correct = group.filter((row) => row.correct).length;
-    const peakRateUsd = sum((row) => row.peakRateUsd), reservedUnknownUsd = sum((row) => row.reservedUnknownUsd);
-    const promptTokens = sum((row) => row.usage.prompt_tokens), cachedTokens = sum((row) => row.usage.cache_read_input_tokens);
-    return { policy, planned: schedule.length, attempted: group.length, missing: schedule.length - group.length,
-      independentTasks: new Set(schedule.map((job) => job.id)).size, correct,
-      accuracy: group.length ? correct / group.length : null, complete: group.length === schedule.length,
-      plannedSolvedFraction: schedule.length ? correct / schedule.length : null,
-      peakRateUsd, reservedUnknownUsd, allAttemptCostPerSuccessUsd: correct ? (peakRateUsd + reservedUnknownUsd) / correct : null,
-      promptTokens, cachedTokens, outputTokens: sum((row) => row.usage.output_tokens),
-      cacheHitRate: promptTokens ? cachedTokens / promptTokens : null,
-      requests: sum((row) => row.requests), cuts: sum((row) => row.cuts), incompleteHandoffs: sum((row) => row.incompleteHandoffs),
-      wallMs: sum((row) => row.wallMs),
-      tasks: [...new Set(schedule.map((job) => job.id))].map((id) => ({ id,
-        planned: schedule.filter((job) => job.id === id).length,
-        attempted: group.filter((row) => row.id === id).length,
-        correct: group.filter((row) => row.id === id && row.correct).length })),
-    };
-  });
-}

package/src/core/query-eval-contract.ts

@@ -1,14 +0,0 @@
-/** Q.21: task-specific shape advice cannot reveal the independent numeric oracle. */
-import type { ToolModeTask } from "./tool-mode-protocol.ts";
-
-export function queryEvalContract(task: ToolModeTask, paths: readonly string[], answerSha256: string) {
-  const columns = task === "tool-small" ? ["amount"] : task === "tool-batch" ? ["total"] : ["team", "total"];
-  const rowCount = task === "tool-aggregate" ? 4 : 1;
-  const operation = task === "tool-small" ? "select amount; return one row with only amount" : task === "tool-batch"
-    ? "sum amount across all records as total; return one row with only total per source"
-    : "keep enabled=true, group by team, sum amount as total, and sort by team ascending; return four rows with columns team,total in that order";
-  const instruction = "Use query on each of " + paths.join(", ") + " to " + operation +
-    '. Write compact JSON with a trailing newline to answer.txt: [{"path":"data-0.jsonl","rows":[...]}], one item per source in the listed order. ' +
-    "Preserve all fixture inputs and run bun run gate.";
-  return { instruction, columns, rowCount, paths: [...paths], answerSha256 };
-}