@andorra-labs/code 0.0.0-stage → 0.1.5

package/scripts/jev-efficiency-live.mjs

@@ -0,0 +1,193 @@
+#!/usr/bin/env node
+// Optional paired live evaluation of the efficiency switches against the configured provider.
+//
+//   ANDORRA_LIVE_EVAL=1 ANDORRA_LIVE_EVAL_HOME=<signed-in evaluation home> node scripts/jev-efficiency-live.mjs [--pairs N] [--feature <name>|all] [--task-file <file>] [--budget-minutes M] [--out <dir>]
+//
+// Runs the built CLI headlessly (`node dist/src/cli.js --prompt ... --json --yolo`) in matched pairs:
+// a baseline run and a candidate run with one efficiency switch (or every switch) turned on, each
+// from a fresh copy of the same empty workspace, with the same provider, model, permissions and
+// limits. The order alternates within each pair to spread warm-up and cache effects. Every run's
+// result, run log and output tree are kept under the output directory; nothing is averaged away.
+//
+// The runs use one dedicated evaluation home, named by ANDORRA_LIVE_EVAL_HOME, for every run. The
+// hosted decision service only answers a signed-in account, and credentials are scoped to the exact
+// home they were created under (`credentialIdentity` in src/auth/AuthManager.ts), so a throwaway
+// home would run every pair anonymously, with the decision manager disabled, and measure the
+// prepared-continuation and postflight switches as if they did nothing. The evaluation home must
+// therefore be signed in on its own (`ANDORRACODE_HOME=<home> andorra login`), with its own
+// evaluation account rather than the user's day-to-day one; the script never copies credentials
+// between homes. Only the home's `efficiency` block changes between runs, and it is restored
+// afterwards. Before any pair is spent, one probe run confirms that decisions are actually
+// requested under this home; without that evidence the script stops and reports why. Every child
+// process — the probe, each paired CLI run and its executable check — is terminated at the
+// remaining evaluation deadline, and a run the deadline killed is recorded as timed out.
+//
+// It refuses to run without ANDORRA_LIVE_EVAL=1, the evaluation home and a configured provider —
+// and, when measuring `adaptiveRouting`, with that switch enabled and its tiers resolvable in the
+// home, because the candidate is the home's own routing configuration. It writes no claim it did
+// not measure, and never fabricates a result for a run that failed. Quality
+// is judged by the executable check the task file names (`check` field), not by the model's own
+// report.
+import { cp, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { spawn } from 'node:child_process';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const args = process.argv.slice(2);
+const option = (name, fallback) => { const index = args.indexOf(name); return index >= 0 ? args[index + 1] : fallback; };
+if (process.env.ANDORRA_LIVE_EVAL !== '1') {
+  process.stderr.write('Live evaluation is opt-in: set ANDORRA_LIVE_EVAL=1. It spends provider credit and time.\n');
+  process.exit(2);
+}
+const evalHome = process.env.ANDORRA_LIVE_EVAL_HOME;
+if (!evalHome) {
+  process.stderr.write('Set ANDORRA_LIVE_EVAL_HOME to a dedicated evaluation home that is signed in (ANDORRACODE_HOME=<home> andorra login) with an evaluation account. The user\'s own home is not used, and credentials are never copied between homes.\n');
+  process.exit(2);
+}
+// `fileURLToPath` decodes the percent escapes a raw `URL.pathname` keeps, so a checkout path with
+// spaces resolves to the directory that exists instead of its percent-encoded ghost.
+const root = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const configPath = join(evalHome, 'config.json');
+const originalConfigText = await readFile(configPath, 'utf8').catch(() => undefined);
+if (originalConfigText === undefined) { process.stderr.write(`${configPath} does not exist; the evaluation home must be configured first.\n`); process.exit(2); }
+const config = JSON.parse(originalConfigText);
+if (!config.providers?.length) { process.stderr.write('No provider is configured in the evaluation home; nothing to evaluate.\n'); process.exit(2); }
+if (config.jevEnabled === false) { process.stderr.write('jevEnabled is false in the evaluation home; the decision-dependent switches cannot be measured.\n'); process.exit(2); }
+const pairs = Number(option('--pairs', '2'));
+const feature = option('--feature', 'all');
+const budgetMinutes = Number(option('--budget-minutes', '30'));
+const out = resolve(option('--out', join(root, 'tests/jev-efficiency/results/live')));
+const taskFile = option('--task-file');
+const tasks = taskFile ? JSON.parse(await readFile(taskFile, 'utf8')) : [
+  { id: 'greenfield-page', prompt: 'Create index.html: a single-file page with a heading "Hello" and a button that toggles the text between "Hello" and "Goodbye". Do not use any build step. Verify the file parses by running node -e "require(\'fs\').readFileSync(\'index.html\',\'utf8\')" and report what you created.', check: 'node -e "const s=require(\'fs\').readFileSync(\'index.html\',\'utf8\'); if(!/<button/i.test(s)||!/Hello/.test(s)) process.exit(1)"' },
+];
+const FEATURES = ['preparedContinuations', 'postflightOnlyWithCandidates', 'toolResultFiltering', 'contextSelection', 'catalogSelection'];
+// Adaptive routing is measured with the evaluation home's own configuration: its tiers and prices
+// are the thing under test, and a synthetic `{ adaptiveRouting: true }` would be rejected outright.
+const homeRouting = config.efficiency?.adaptiveRouting;
+const routingUsable = !!homeRouting && typeof homeRouting === 'object' && homeRouting.enabled === true
+  && !!homeRouting.tiers && Object.keys(homeRouting.tiers).length > 0;
+if (feature === 'adaptiveRouting' && !routingUsable) {
+  process.stderr.write('Measuring adaptiveRouting requires the evaluation home to configure efficiency.adaptiveRouting with enabled: true and tiers; the script measures that configuration as the candidate.\n');
+  process.exit(2);
+}
+if (routingUsable && (feature === 'adaptiveRouting' || feature === 'all')) {
+  for (const [tier, target] of Object.entries(homeRouting.tiers)) {
+    const provider = (config.providers ?? []).find(item => item?.id === target?.provider);
+    const model = target?.model ?? provider?.model;
+    if (!provider || !model || (target.model !== undefined && provider.models?.length && !provider.models.includes(target.model) && provider.model !== target.model)) {
+      process.stderr.write(`efficiency.adaptiveRouting.tiers.${tier} names ${target?.provider ?? '?'}/${target?.model ?? '?'} which the evaluation home does not configure; routing would treat it as unconfigured and the measurement would be void.\n`);
+      process.exit(2);
+    }
+  }
+}
+const candidateConfig = feature === 'all'
+  ? { ...Object.fromEntries(FEATURES.map(name => [name, true])), ...(routingUsable ? { adaptiveRouting: homeRouting } : {}) }
+  : feature === 'adaptiveRouting' ? { adaptiveRouting: homeRouting } : { [feature]: true };
+// Prices belong to both variants: the runtime's accounting reads them from the efficiency block, so a
+// baseline run without the block would report every cost as unavailable and the pair could not
+// compare spending. The baseline carries the prices with routing off; a candidate that does not
+// measure routing carries them the same way.
+const pricing = homeRouting && typeof homeRouting === 'object' && homeRouting.pricing ? homeRouting.pricing : undefined;
+const baselineConfig = pricing ? { adaptiveRouting: { enabled: false, pricing } } : undefined;
+if (pricing && !candidateConfig.adaptiveRouting) candidateConfig.adaptiveRouting = { enabled: false, pricing };
+
+// One child process under the remaining evaluation budget: the spawn `timeout` terminates it at
+// the deadline and `close` still awaits it, so a hung run or check can hold neither the budget
+// nor the configuration restoration below. `signal` is set when the deadline killed it.
+const run = (command, commandArgs, options, timeoutMs) => new Promise(resolveRun => {
+  const child = spawn(command, commandArgs, { ...options, windowsHide: true, ...(Number.isFinite(timeoutMs) && timeoutMs > 0 ? { timeout: timeoutMs } : {}) });
+  let stdout = '', stderr = '';
+  child.stdout.on('data', chunk => { stdout += chunk; });
+  child.stderr.on('data', chunk => { stderr += chunk; });
+  child.on('close', (code, signal) => resolveRun({ code, stdout, stderr, ...(signal ? { signal } : {}) }));
+});
+const runLogs = async () => new Set((await readdir(join(evalHome, 'runs')).catch(() => [])).filter(name => name.endsWith('.jsonl')));
+const eventTypes = async names => {
+  const types = [];
+  for (const name of names) for (const line of (await readFile(join(evalHome, 'runs', name), 'utf8')).split('\n')) { if (!line.trim()) continue; try { types.push(JSON.parse(line).type); } catch { /* partial line */ } }
+  return types;
+};
+
+// One CLI run under the evaluation home with the given efficiency block; returns the outcome and the
+// run logs it produced. The home's config is rewritten for the run and restored by the caller.
+async function evaluate(task, efficiency, label) {
+  if (deadline - Date.now() <= 0) return { exitCode: null, durationMs: 0, checkPassed: null, usage: null, status: null, decisionsRequested: 0, efficiencyEvents: 0, timedOut: true };
+  await writeFile(configPath, JSON.stringify({ ...config, ...(efficiency ? { efficiency } : {}) }, null, 2));
+  if (!efficiency) { const cleared = { ...config }; delete cleared.efficiency; await writeFile(configPath, JSON.stringify(cleared, null, 2)); }
+  const workspace = await mkdtemp(join(tmpdir(), 'andorra-live-eval-ws-'));
+  const before = await runLogs();
+  try {
+    const remainingMs = deadline - Date.now();
+    const startedAt = Date.now();
+    const cli = await run(process.execPath, [join(root, 'dist/src/cli.js'), '--cwd', workspace, '--prompt', task.prompt, '--json', '--yolo'], { cwd: workspace, env: { ...process.env, ANDORRACODE_HOME: evalHome } }, remainingMs);
+    const durationMs = Date.now() - startedAt;
+    // A run the deadline killed still reports its partial output; the executable check is not
+    // spent on a CLI run that never finished.
+    const timedOut = cli.signal !== undefined;
+    const check = !timedOut && task.check ? await run(process.platform === 'win32' ? 'cmd' : 'sh', process.platform === 'win32' ? ['/c', task.check] : ['-c', task.check], { cwd: workspace }, deadline - Date.now()) : undefined;
+    let summary;
+    try { summary = JSON.parse(cli.stdout.trim().split('\n').at(-1)); } catch { summary = undefined; }
+    // The runtime applies the home's configured routing prices to its own usage, so the run
+    // summary's per-model tuples carry the cost. The capture's completeness rule applies here too:
+    // a tuple counts only when every request in it was priced, and the run reports a total only
+    // when every model it used was priced — a partial sum would quietly omit a model's spend.
+    const modelTuples = summary?.telemetry?.models ?? [];
+    const costMicroUsd = modelTuples.length && modelTuples.every(tuple => typeof tuple.costMicroUsd === 'number' && tuple.reported?.cost === tuple.requests)
+      ? modelTuples.reduce((total, tuple) => total + tuple.costMicroUsd, 0)
+      : null;
+    const produced = [...await runLogs()].filter(name => !before.has(name));
+    await mkdir(join(out, label), { recursive: true });
+    await writeFile(join(out, label, 'cli-stdout.json'), cli.stdout);
+    await writeFile(join(out, label, 'cli-stderr.txt'), cli.stderr);
+    for (const name of produced) await cp(join(evalHome, 'runs', name), join(out, label, 'runs', name)).catch(() => {});
+    await cp(workspace, join(out, label, 'output'), { recursive: true }).catch(() => {});
+    const types = await eventTypes(produced);
+    return { exitCode: cli.code, durationMs, checkPassed: check ? check.code === 0 : null, usage: summary?.usage ?? null, costMicroUsd, status: summary?.status ?? null, decisionsRequested: types.filter(type => type === 'decision.requested').length, efficiencyEvents: types.filter(type => type.startsWith('efficiency.')).length, ...(timedOut ? { timedOut: true } : {}) };
+  } finally {
+    await rm(workspace, { recursive: true, force: true });
+  }
+}
+
+const started = Date.now();
+// The whole evaluation, probe included, ends by this clock: every child process it spawns is
+// handed the remaining allowance as its timeout, so no single run can spend past the budget.
+const deadline = started + budgetMinutes * 60_000;
+await mkdir(out, { recursive: true });
+const results = [];
+let aborted;
+try {
+  // The probe: a candidate run must show the decision service answering under this home, or the
+  // decision-dependent switches would be measured while disabled and the comparison would be void.
+  const probe = await evaluate(tasks[0], candidateConfig, 'probe-candidate');
+  if (probe.timedOut) {
+    aborted = `the probe run hit the ${budgetMinutes}-minute evaluation budget before it finished; no pair was spent. Raise --budget-minutes or shorten the task.`;
+  } else if (probe.decisionsRequested === 0) {
+    aborted = `the probe run under ${evalHome} requested no decision (exit ${probe.exitCode}, status ${probe.status}); the home is not signed in to the decision service or decisions are disabled, so the comparison would measure disabled switches. Sign in under this home (ANDORRACODE_HOME=${evalHome} andorra login) and retry.`;
+  }
+  for (let pair = 1; pair <= pairs && !aborted; pair++) {
+    for (const task of tasks) {
+      const order = pair % 2 === 1 ? ['baseline', 'candidate'] : ['candidate', 'baseline'];
+      for (const variant of order) {
+        if (Date.now() - started > budgetMinutes * 60_000) { process.stderr.write('Time budget exhausted; stopping.\n'); break; }
+        const label = `${task.id}-pair${pair}-${variant}`;
+        const outcome = await evaluate(task, variant === 'candidate' ? candidateConfig : baselineConfig, label);
+        results.push({ task: task.id, pair, variant, order: order.indexOf(variant) + 1, label, ...outcome });
+      }
+    }
+  }
+} finally {
+  await writeFile(configPath, originalConfigText);
+}
+const report = { feature, candidateConfig, evaluationHome: evalHome, pairs, sampleSize: results.length, aborted: aborted ?? null, results, caveats: [
+  'Each run is a separate process against the live provider; run-to-run variation is not attributable to the efficiency switches alone.',
+  'Prompt-cache state is not controlled beyond alternating order within a pair.',
+  'Usage figures are provider-reported where present; a run that reports none is left null, not zero.',
+  'costMicroUsd is the evaluation home\'s configured efficiency.adaptiveRouting.pricing applied by the runtime to its own usage; a run whose models were not all priced reports null, not a partial sum.',
+  'decisionsRequested counts decision.requested events in the run log; a candidate run with zero requested decisions did not exercise the decision-dependent switches.',
+] };
+await writeFile(join(out, 'summary.json'), JSON.stringify(report, null, 2) + '\n');
+if (aborted) { process.stderr.write(`Aborted before the paired runs: ${aborted}\n`); process.exit(3); }
+process.stdout.write(`${results.length} run(s) recorded under ${out}\n`);
+for (const row of results) process.stdout.write(`${row.label}: exit ${row.exitCode}, status ${row.status}, check ${row.checkPassed}, ${row.durationMs} ms, decisions ${row.decisionsRequested}, usage ${JSON.stringify(row.usage)}, cost ${row.costMicroUsd === null || row.costMicroUsd === undefined ? 'unavailable' : `$${(row.costMicroUsd / 1_000_000).toFixed(4)}`}${row.timedOut ? ', killed at the evaluation deadline' : ''}\n`);

package/scripts/zcode-engine-replay.mjs

@@ -0,0 +1,223 @@
+// Matched, offline historical-payload fixture. Never calls a configured provider.
+import assert from 'node:assert/strict';
+import { createReadStream } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { createHash } from 'node:crypto';
+import { resolve, join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+const [checkout = '.', trajectory, countArgument = '100'] = process.argv.slice(2);
+if (!trajectory) throw new Error('Usage: node scripts/zcode-engine-replay.mjs <built-checkout> <V3-trajectory-jsonl> [tool-response-count=100]');
+const count = Number(countArgument);
+if (!Number.isInteger(count) || count < 20 || count > 200) throw new Error('Choose 20–200 historical tool responses');
+const root = resolve(checkout);
+const { AndorraAgentEngine } = await import(pathToFileURL(join(root, 'dist/src/hardened-runtime.js')).href);
+const invocations = new Map();
+const sourceHash = createHash('sha256');
+const input = createReadStream(trajectory);
+input.on('data', chunk => sourceHash.update(chunk));
+let runId;
+let invocationId;
+let ordinal = 0;
+for await (const line of createInterface({ input })) {
+  if (!line.trim()) continue;
+  const event = JSON.parse(line);
+  event.ordinal = ordinal++;
+  if (!['model.request', 'model.response', 'runtime.event'].includes(event.type)) continue;
+  if (event.type === 'runtime.event' && !['node.tool.completed', 'node.tool.failed'].includes(event.event?.type)) continue;
+  const eventRunId = event.type === 'runtime.event' ? event.event?.data?.runId : event.runId;
+  if (runId !== undefined && eventRunId !== runId) continue;
+  if (typeof eventRunId !== 'string' || !eventRunId) continue;
+  const identity = event.type === 'runtime.event' ? event.event?.data : event;
+  const eventInvocationId = identity?.invocationId;
+  const eligible = event.type === 'model.response' && event.toolCalls?.length
+    && !event.toolCalls.some(call => ['complete_agent', 'update_todos'].includes(call.name));
+  if (invocationId !== undefined && eventInvocationId !== invocationId) continue;
+  if (typeof eventInvocationId !== 'string' || !eventInvocationId) {
+    assert.ok(!eligible, 'Exact visible-payload replay is unavailable: legacy evidence lacks an unambiguous agent invocation identity');
+    continue;
+  }
+  const key = JSON.stringify([eventRunId, eventInvocationId]);
+  let run = invocations.get(key);
+  if (!run) {
+    run = { requests: new Map(), responses: [], observations: new Map(), latestToolResponses: new Map(),
+      identity: JSON.stringify([identity.nodeId ?? null, identity.subagentId ?? null]),
+      recordedUsage: { inputTokens: 0, outputTokens: 0, mainRequests: 0, auxiliaryRequests: 0 } };
+    invocations.set(key, run);
+  }
+  assert.equal(JSON.stringify([identity.nodeId ?? null, identity.subagentId ?? null]), run.identity,
+    'Exact visible-payload replay requires consistent node/subagent identity within an invocation');
+  const { requests, responses, observations, latestToolResponses, recordedUsage } = run;
+  if (event.type === 'model.request') {
+    assert.ok(!requests.has(event.requestId), 'Exact visible-payload replay requires unique request ids within its invocation');
+    requests.set(event.requestId, event);
+  }
+  if (eligible) {
+    responses.push(event);
+    runId ??= event.runId;
+    invocationId ??= eventInvocationId;
+    // Request and tool ids may repeat across graph nodes, retries and copies.
+    for (const otherKey of invocations.keys()) if (otherKey !== key) invocations.delete(otherKey);
+  }
+  if (event.type === 'model.response') {
+    // Call ids can recur later in the same run. Bind status events to their
+    // preceding response occurrence, including responses outside the selection.
+    for (const call of event.toolCalls ?? []) latestToolResponses.set(call.id, event.ordinal);
+    recordedUsage.inputTokens += event.usage?.inputTokens ?? 0;
+    recordedUsage.outputTokens += event.usage?.outputTokens ?? 0;
+    if (requests.get(event.requestId)?.tools?.length) recordedUsage.mainRequests++;
+    else recordedUsage.auxiliaryRequests++;
+  }
+  if (event.type === 'runtime.event' && ['node.tool.completed', 'node.tool.failed'].includes(event.event.type)) {
+    const data = event.event.data;
+    // Runtime outputs are public projections, not necessarily model observations.
+    const responseOrdinal = latestToolResponses.get(data.toolCallId);
+    if (responseOrdinal !== undefined) {
+      const batch = observations.get(responseOrdinal) ?? new Map();
+      const recorded = batch.get(data.toolCallId);
+      if (recorded) recorded.statusCount++;
+      else batch.set(data.toolCallId, { failed: event.event.type === 'node.tool.failed', statusCount: 1 });
+      observations.set(responseOrdinal, batch);
+    }
+  }
+}
+assert.ok(runId !== undefined, 'Historical fixture must contain a tool-bearing response with a run id');
+const { requests, responses, observations: responseObservations, recordedUsage } = invocations.get(JSON.stringify([runId, invocationId]));
+const observations = new Map();
+assert.ok(responses.length >= count, 'Historical fixture must contain the requested calls');
+const selected = responses.slice(0, count);
+const firstRequest = requests.get(selected[0].requestId);
+assert.ok(firstRequest, 'The selected invocation must contain the request for its first selected response');
+const expectedCalls = selected.reduce((total, response) => total + response.toolCalls.length, 0);
+const assertRecoverable = value => assert.ok(!JSON.stringify(value).includes('[REDACTED'), 'Exact visible-payload replay is unavailable: selected evidence was sanitized');
+assertRecoverable(firstRequest.messages.filter(message => message.role === 'user'));
+const schemas = new Map();
+const selectedIds = new Set();
+const observationHash = createHash('sha256');
+const matchingCalls = (left, right) => left?.length === right.length && left.every((call, index) =>
+  call.id === right[index].id && call.name === right[index].name && call.arguments === right[index].arguments);
+for (const response of selected) {
+  const matchingRequest = requests.get(response.requestId);
+  assert.ok(matchingRequest && matchingRequest.ordinal < response.ordinal, 'Every selected response must have its matching preceding request');
+  assertRecoverable({ text: response.text, toolCalls: response.toolCalls });
+  // Locate the first subsequent request carrying this exact assistant exchange.
+  // Matching the entire batch avoids borrowing feedback from another invocation.
+  let feedback;
+  for (const subsequent of requests.values()) {
+    if (subsequent.ordinal <= response.ordinal) continue;
+    const messages = subsequent.messages ?? [];
+    const assistantIndex = messages.findIndex(message => message.role === 'assistant' && matchingCalls(message.toolCalls, response.toolCalls));
+    if (assistantIndex < 0) continue;
+    const batch = messages.slice(assistantIndex + 1, assistantIndex + 1 + response.toolCalls.length);
+    if (batch.length === response.toolCalls.length && batch.every(message => message.role === 'tool')
+      && response.toolCalls.every(call => batch.filter(message => message.toolCallId === call.id).length === 1)) {
+      feedback = batch;
+      break;
+    }
+  }
+  assert.ok(feedback, 'Exact visible-payload replay is unavailable: no subsequent matching model request contains complete tool feedback');
+  for (const call of response.toolCalls) {
+    assert.ok(!selectedIds.has(call.id), 'Exact visible-payload replay requires unique selected tool call ids within its invocation');
+    selectedIds.add(call.id);
+    const schema = matchingRequest.tools?.find(tool => tool.name === call.name);
+    assert.ok(schema, 'Every historical call must be advertised by its own matching request');
+    assertRecoverable(schema);
+    const existing = schemas.get(call.name);
+    if (existing) {
+      assert.ok(JSON.stringify(existing.parameters) === JSON.stringify(schema.parameters)
+        && existing.executionMode === schema.executionMode, 'Exact replay does not support changing schemas for the same tool name');
+    } else schemas.set(call.name, schema);
+    const message = feedback.find(message => message.toolCallId === call.id);
+    assert.ok(typeof message.content === 'string', 'Exact visible-payload replay requires recorded textual feedback');
+    assertRecoverable(message.content);
+    // Trajectories intentionally omit native images. Never reconstruct or expose them.
+    assert.ok(!message.images?.length && !/"screenshotId"\s*:/.test(message.content), 'Exact visible-payload replay is unavailable: native image evidence is omitted or unsupported');
+    const recorded = responseObservations.get(response.ordinal)?.get(call.id);
+    assert.ok(recorded, 'Every selected historical call has a recorded completion status');
+    assert.equal(recorded.statusCount, 1, 'Exact visible-payload replay requires one unambiguous completion status for each selected call occurrence');
+    recorded.output = message.content;
+    observations.set(call.id, recorded);
+    observationHash.update(JSON.stringify([call.id, message.content]) + '\n');
+  }
+}
+const names = new Set(selected.flatMap(response => response.toolCalls.map(call => call.name)));
+const queues = new Map();
+const keyFor = (name, args) => name + ':' + JSON.stringify(args);
+for (const response of selected) for (const call of response.toolCalls) {
+  const args = JSON.parse(call.arguments);
+  const key = keyFor(call.name, args);
+  const queue = queues.get(key) ?? [];
+  queue.push(observations.get(call.id));
+  queues.set(key, queue);
+}
+let executed = 0, recordedFailures = 0;
+const tools = [...names].map(name => {
+  const schema = schemas.get(name);
+  assert.ok(schema, 'Only actually advertised historical tools are replayed');
+  return { name, description: schema.description, parameters: schema.parameters, executionMode: schema.executionMode ?? 'sequential',
+    execute: async args => {
+      const recorded = queues.get(keyFor(name, args))?.shift();
+      assert.ok(recorded, 'Scripted tool arguments match the selected historical payload');
+      executed++;
+      if (recorded.failed) { recordedFailures++; throw new Error(recorded.output.replace(/^Tool error:\s*/, '')); }
+      return recorded.output;
+    },
+  };
+});
+let cursor = 0, mainRequests = 0, auxiliaryRequests = 0;
+let mainMessageBytes = 0, auxiliaryMessageBytes = 0, replayedArgumentBytes = 0;
+const events = [];
+const failedIds = new Set();
+const marker = 'OFFLINE_FIXTURE_CHECKPOINT';
+const result = await new AndorraAgentEngine().execute({
+  task: firstRequest.messages.filter(message => message.role === 'user').map(message => message.content).join('\n'),
+  provider: 'offline', model: 'historical-payload-fixture', contextWindow: 128_000,
+  subagentId: 'offline-payload-fixture', tools,
+  limits: { maxIterations: count + 20, maxToolCalls: expectedCalls + 20, maxContextChars: 2_000_000 },
+  context: { workspace: root, signal: new AbortController().signal, yolo: true, approve: async () => { throw new Error('No real mutations or approvals occur in this fixture'); } },
+  onEvent: event => {
+    if (event.type === 'node.tool.failed') failedIds.add(event.data.toolCallId);
+    if (['agent.context.compacted', 'agent.context.microcompacted', 'agent.context.compaction.started', 'agent.session.completed'].includes(event.type)) events.push({ type: event.type, data: event.data });
+  },
+  adapter: { generate: async request => {
+    const bytes = Buffer.byteLength(JSON.stringify(request.messages));
+    if (!request.tools.length) {
+      auxiliaryRequests++;
+      auxiliaryMessageBytes += bytes;
+      // Constant fake summary isolates retirement/prefix mechanics. Its quality
+      // is neither a historical summary nor a measured model-generated result.
+      return { text: marker + ': Historical scripted prefix retired. Continue the fixed replay; task quality is not evaluated.', toolCalls: [] };
+    }
+    mainRequests++;
+    mainMessageBytes += bytes;
+    if (cursor > 0) for (const call of selected[cursor - 1].toolCalls) {
+      const message = request.messages.find(message => message.role === 'tool' && message.toolCallId === call.id);
+      assert.ok(message?.content === observations.get(call.id).output, 'Exact visible-payload replay failed: engine feedback differs from the recorded model observation');
+    }
+    for (const message of request.messages) for (const call of message.toolCalls ?? []) replayedArgumentBytes += Buffer.byteLength(call.arguments);
+    assert.ok(request.messages.some(message => message.role === 'user' && message.content.includes('Begin implementation now')), 'Original task stays pinned');
+    const callIds = request.messages.flatMap(message => message.toolCalls ?? []).map(call => call.id).sort();
+    assert.deepEqual(request.messages.filter(message => message.role === 'tool').map(message => message.toolCallId).sort(), callIds, 'Every tool call/result pair remains intact');
+    if (cursor === selected.length) return { text: JSON.stringify({ summary: 'Offline historical payload fixture completed; original task was not rerun.', tests: [] }), toolCalls: [] };
+    const response = selected[cursor++];
+    return { text: response.text ?? '', toolCalls: response.toolCalls.map(call => ({ ...call })) };
+  } },
+});
+assert.equal(result.status, 'succeeded', result.reason);
+assert.equal(cursor, selected.length);
+assert.equal(mainRequests, count + 1);
+assert.ok([...queues.values()].flat().every(recorded => recorded.failed), 'Only historically invalid calls can be rejected before fake execution');
+assert.deepEqual([...failedIds].sort(), selected.flatMap(response => response.toolCalls).filter(call => observations.get(call.id).failed).map(call => call.id).sort(), 'Recorded failures remain failures; the fixture does not conceal historical invalid calls');
+process.stdout.write(JSON.stringify({
+  fixture: 'matched offline historical visible-payload replay; no provider-token or task-quality measurement',
+  sourceSha256: sourceHash.digest('hex'), sourceRunId: runId, sourceInvocationId: invocationId,
+  modelVisibleObservationSha256: observationHash.digest('hex'),
+  historicalRecordedUsage: { ...recordedUsage, totalTokens: recordedUsage.inputTokens + recordedUsage.outputTokens },
+  firstSelectedResponse: selected[0].time, lastSelectedResponse: selected.at(-1).time,
+  scriptedHistoricalResponses: count, scriptedHistoricalToolCalls: expectedCalls,
+  executedToolObservations: executed, hostRejectedBeforeFakeExecution: expectedCalls - executed, recordedToolFailures: recordedFailures,
+  status: result.status, mainRequests, auxiliaryRequests,
+  cumulativeMainMessageBytes: mainMessageBytes, cumulativeAuxiliaryMessageBytes: auxiliaryMessageBytes,
+  cumulativeToolArgumentReplayBytes: replayedArgumentBytes,
+  contextEvents: events.map(event => ({ type: event.type, ...Object.fromEntries(Object.entries(event.data).filter(([key]) => ['executor', 'removedExchanges', 'contextChars', 'reason', 'beforeTokens', 'afterTokens', 'threshold', 'observations', 'removedChars', 'batch'].includes(key))) })),
+}, null, 2) + '\n');