agenticreplay 0.1.0 → 0.1.2

package/scripts/conformance.mjs

@@ -1,245 +0,0 @@
-#!/usr/bin/env node
-/**
- * Trace format conformance runner.
- *
- * Validates every trace under examples/traces/ against the normative JSON Schema, and validates a
- * trace produced by our own writer alongside them. Both halves are load-bearing and they prove
- * different things: a hand-written example validating shows the schema is targetable by an
- * implementation that is not ours, and the writer's output validating shows *ours* still conforms.
- *
- * Checking only the examples — which is what this did — let the shipped fixture drift into
- * containing events the writer cannot produce while the job reported a clean bill of health for
- * "the format". So the run also reports which declared event types no shipped trace exercises,
- * because a type nothing emits and nothing tests is a claim, not a feature.
- */
-import { mkdtemp, readdir, readFile, rm, stat } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { EVENT_TYPES, validateEvent, validateManifest } from '../packages/schema/dist/index.js';
-import { TraceWriter } from '../packages/core/dist/index.js';
-
-const root = fileURLToPath(new URL('..', import.meta.url));
-const examplesDir = join(root, 'examples', 'traces');
-
-let checked = 0;
-let failures = 0;
-/** Every event type any validated trace actually contained. */
-const seen = new Set();
-
-function fail(where, errors) {
-  failures += 1;
-  console.error(`FAIL ${where}`);
-  for (const e of errors) console.error(`     ${e}`);
-}
-
-async function checkTrace(dir, label) {
-  const manifest = JSON.parse(await readFile(join(dir, 'manifest.json'), 'utf8'));
-  const mr = validateManifest(manifest);
-  if (!mr.valid) fail(`${dir}/manifest.json`, mr.errors);
-
-  const raw = await readFile(join(dir, 'events.jsonl'), 'utf8');
-  const lines = raw.split('\n').filter((l) => l.trim().length > 0);
-  let expectedSeq = 0;
-  for (const [i, line] of lines.entries()) {
-    let event;
-    try {
-      event = JSON.parse(line);
-    } catch {
-      // A truncated final line is legal (spec §2); anything earlier is not.
-      if (i === lines.length - 1) continue;
-      fail(`${dir}/events.jsonl:${i + 1}`, ['unparseable line']);
-      continue;
-    }
-    const r = validateEvent(event);
-    if (!r.valid) fail(`${dir}/events.jsonl:${i + 1}`, r.errors);
-    if (event.seq !== expectedSeq) {
-      fail(`${dir}/events.jsonl:${i + 1}`, [`seq ${event.seq}, expected dense ${expectedSeq}`]);
-    }
-    expectedSeq = event.seq + 1;
-    seen.add(event.type);
-    checked += 1;
-  }
-  console.log(`  ${label ?? dir.replace(root, '')}: ${lines.length} events`);
-}
-
-let entries = [];
-try {
-  entries = await readdir(examplesDir);
-} catch {
-  console.log('no examples/traces/ directory yet — nothing to check');
-  process.exit(0);
-}
-
-for (const name of entries) {
-  const dir = join(examplesDir, name);
-  if (!(await stat(dir)).isDirectory()) continue;
-  await checkTrace(dir);
-}
-
-// The quickstart asset is a real recording that ships inside the npm package and is replayed on
-// every `agenticreplay quickstart`, so it is held to the format the same way the hand-written examples are.
-// It is not under examples/traces/ because it is not a fixture — it is the product's own demo.
-await checkTrace(join(root, 'packages', 'cli', 'quickstart', 'trace'), 'the quickstart trace');
-
-// A trace this repository's writer produced, checked by the same rules as the examples. Without
-// it the job could stay green while the writer emitted something the schema forbids.
-const scratch = await mkdtemp(join(tmpdir(), 'agenticreplay-conformance-'));
-try {
-  const writer = await TraceWriter.create(scratch, {
-    adapter: { id: 'generic-openai', version: '0.0.0' },
-    argv: ['generic-openai'],
-    cwd: scratch,
-    agenticreplayVersion: '0.0.0',
-  });
-  await writer.append({
-    type: 'run.start',
-    actor: 'agenticreplay',
-    turn: 0,
-    attrs: { adapter: 'x' },
-  });
-  await writer.append({
-    type: 'model.request',
-    actor: 'agent',
-    turn: 1,
-    attrs: { model: 'gpt-5.2', dialect: 'openai' },
-    payload: JSON.stringify({ model: 'gpt-5.2', messages: [] }),
-  });
-  await writer.append({
-    type: 'model.response',
-    actor: 'model',
-    turn: 1,
-    attrs: { model: 'gpt-5.2', status: 200, input_tokens: 1, output_tokens: 1 },
-    payload: JSON.stringify({ id: 'x', content: [] }),
-  });
-  // A tunnelled and a decrypted connection, which `--tls-intercept` now emits for real. Without a
-  // pair here the coverage line below still reported `net.*` as unexercised while the recorder was
-  // writing them, which is the opposite of the honesty this report exists to provide.
-  await writer.append({
-    type: 'net.request',
-    actor: 'agent',
-    turn: 1,
-    attrs: { host: 'api.example.com', port: 443, method: 'POST', path: '/v1/x' },
-  });
-  await writer.append({
-    type: 'net.response',
-    actor: 'gateway',
-    turn: 1,
-    attrs: { host: 'api.example.com', status: 200, duration_ms: 12 },
-  });
-  // An MCP pair, which `--mcp-config` emits for real. It reached this list only after the shim
-  // resolution was fixed — until then the recorder rewrote every stdio server to launch a module
-  // that exits without starting it, so this type was unexercised because the layer did not work,
-  // which is exactly what the coverage line below exists to surface.
-  await writer.append({
-    type: 'mcp.request',
-    actor: 'agent',
-    turn: 1,
-    attrs: { server: 'files', kind: 'request', method: 'tools/list', id: 1 },
-    payload: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/list' }),
-  });
-  await writer.append({
-    type: 'mcp.response',
-    actor: 'agent',
-    turn: 1,
-    attrs: { server: 'files', kind: 'response', id: 1 },
-    payload: JSON.stringify({ jsonrpc: '2.0', id: 1, result: { tools: [] } }),
-  });
-  // What a retriever put into a prompt, which `agenticreplay record` derives for real from any request
-  // carrying a `Context: … Question: …` structure — the shape every RAG template writes. Derived
-  // rather than captured, like `tool.call`: the passages are in the request because putting them
-  // there is what retrieval is for.
-  await writer.append({
-    type: 'retrieval.context',
-    actor: 'harness',
-    turn: 1,
-    attrs: {
-      query: 'Where was Marie Curie born?',
-      passages: 2,
-      chars: 96,
-      derived_from: 'prompt',
-    },
-    payload: JSON.stringify({
-      query: 'Where was Marie Curie born?',
-      passages: ['Marie Curie was born in Warsaw in 1867.', 'She later moved to Paris.'],
-    }),
-  });
-  // The harness's own transcript, which `agenticreplay record` captures for real whenever the adapter
-  // knows where the harness keeps one. It is the only record of what a hand-driven run was
-  // *asked*, so a trace missing it cannot be replayed at all — worth a line here for the same
-  // reason the MCP pair above is.
-  await writer.append({
-    type: 'session.snapshot',
-    actor: 'harness',
-    turn: 1,
-    attrs: {
-      harness: 'claude-code',
-      session_id: '11111111-2222-3333-4444-555555555555',
-      rel_path: 'a1b2c3.jsonl',
-      prompts: 2,
-      bytes: 128,
-    },
-    payload: JSON.stringify({ type: 'user', message: { role: 'user', content: 'hello' } }),
-  });
-  // A fork and the divergence a fork reports. Both come out of `agenticreplay replay --from N --model X`.
-  await writer.append({
-    type: 'fork',
-    actor: 'agenticreplay',
-    turn: 1,
-    attrs: { parent_run: 'run_000000000000', fork_point: 2, model: 'gpt-5.2' },
-  });
-  await writer.append({
-    type: 'divergence',
-    actor: 'agenticreplay',
-    turn: 1,
-    attrs: { level: 'minor', rung: 2, detail: 'trailing message differs', source_seq: 2 },
-  });
-  // A routing decision, which a fork emits for real now: agenticreplay substituted the model, picked the
-  // dialect that serves it and picked the origin.
-  await writer.append({
-    type: 'route.decision',
-    actor: 'gateway',
-    turn: 1,
-    attrs: {
-      model: 'gpt-5.2',
-      target: 'openai',
-      recorded: 'anthropic',
-      origin: 'https://api.openai.com',
-      crossProvider: true,
-      reason: 'gpt-5.2 is served by openai, not the recorded anthropic',
-    },
-  });
-  await writer.append({
-    type: 'run.end',
-    actor: 'agenticreplay',
-    turn: 1,
-    attrs: { exit_code: 0 },
-  });
-  await writer.close(0);
-  await checkTrace(writer.runDir, 'this writer, freshly recorded');
-} finally {
-  await rm(scratch, { recursive: true, force: true });
-}
-
-// Only types this repository actually emits are added above. Writing a synthetic event for a type
-// nothing produces would empty this list while making it a lie — the line is worth having precisely
-// because it is what noticed that MCP capture emitted nothing.
-//
-// `checkpoint` is the one type that is *supposed* to stay unexercised: spec §3 derives a checkpoint
-// at read time and forbids recording one. Reporting it beside genuine gaps invites someone to close
-// it by emitting the event the spec says not to — which is exactly what the shipped example used to
-// do — so it is named as intentional instead.
-const DERIVED_ONLY = new Set(['checkpoint']);
-const unexercised = EVENT_TYPES.filter((t) => !seen.has(t) && !DERIVED_ONLY.has(t));
-const derived = EVENT_TYPES.filter((t) => !seen.has(t) && DERIVED_ONLY.has(t));
-if (derived.length > 0) {
-  console.log(`\nderived at read time, never recorded (spec §3): ${derived.join(', ')}`);
-}
-console.log(
-  unexercised.length > 0
-    ? `declared but not exercised by any trace here: ${unexercised.join(', ')}`
-    : 'every declared event type is exercised by a trace here',
-);
-
-console.log(`\n${checked} events checked, ${failures} failure(s)`);
-process.exit(failures === 0 ? 0 : 1);

package/scripts/fidelity.mjs

@@ -1,170 +0,0 @@
-#!/usr/bin/env node
-/**
- * Replay fidelity scoreboard.
- *
- * "Replay is reliable" is the load-bearing claim of this project and it has been prose. Prose does
- * not regress visibly: a change to the matcher, a dialect, or the redactor can quietly turn an
- * exact replay into a lossy one, and nothing says so until someone is debugging at 2am with a
- * trace that no longer reproduces.
- *
- * So: replay a corpus of real recordings, assert the numbers, and write them down. A run is judged
- * on three counts the replay already reports —
- *
- *   unmatched   turns the recording could not serve at all. The budget for this is zero. A turn
- *               agenticreplay cannot answer from the trace is the failure the whole feature exists to
- *               prevent, and there is no honest way to spend a budget on it.
- *   canonical_exact turns matched after canonical normalization. Below `minExact` is a regression even when nothing
- *               is unmatched, because a run that only matches loosely is a run whose fidelity is
- *               being carried by the match ladder rather than by the capture.
- *   divergences reported differences. Allowed a budget, because some are inherent: a harness that
- *               stamps a fresh session id into every request diverges by construction.
- *
- * Each entry names its own budget rather than sharing one. A corpus with a single global
- * threshold is a corpus where the worst entry sets the bar for all of them.
- *
- *   node scripts/fidelity.mjs           # check, and write docs/fidelity.md
- *   node scripts/fidelity.mjs --check   # check only; fail on regression, write nothing
- */
-import { execFile } from 'node:child_process';
-import { cp, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { dirname, join } from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { promisify } from 'node:util';
-
-const exec = promisify(execFile);
-const root = fileURLToPath(new URL('..', import.meta.url));
-const cli = join(root, 'packages', 'cli', 'dist', 'cli.js');
-
-/**
- * The corpus.
- *
- * One entry per harness whose capture this project claims to support. Each names a trace, the
- * project state that trace was recorded against, and what its replay is expected to achieve.
- * Adding a harness to the README's support table without adding it here is how "we support X"
- * goes back to being a sentence.
- */
-const CORPUS = [
-  {
-    id: 'node-agent',
-    harness: 'a Node agent over chat completions',
-    trace: join(root, 'packages', 'cli', 'quickstart', 'trace'),
-    project: join(root, 'packages', 'cli', 'quickstart', 'project'),
-    note: 'the run `agenticreplay quickstart` ships and replays on every invocation',
-    budget: { minExact: 3, maxDivergences: 0 },
-  },
-];
-
-/** Replay one entry in a scratch copy and read the numbers back out of the JSON. */
-async function replay(entry) {
-  const dir = await mkdtemp(join(tmpdir(), 'agenticreplay-fidelity-'));
-  try {
-    const manifest = JSON.parse(await readFile(join(entry.trace, 'manifest.json'), 'utf8'));
-    await cp(entry.project, dir, { recursive: true });
-    await cp(entry.trace, join(dir, '.agenticreplay', 'runs', manifest.run_id), {
-      recursive: true,
-    });
-
-    // `--in-place` because the project is copied in at the state the run started from; restoring
-    // would mean writing over a directory the recording never saw.
-    // A halted replay exits non-zero and still prints its numbers, which are exactly the numbers
-    // worth reporting — "it did not complete" hides whether one turn was unmatched or twenty.
-    let stdout;
-    try {
-      ({ stdout } = await exec(
-        process.execPath,
-        [cli, 'replay', manifest.run_id, '--in-place', '--json'],
-        { cwd: dir, env: { ...process.env, NO_COLOR: '1' } },
-      ));
-    } catch (err) {
-      stdout = (err && typeof err === 'object' && 'stdout' in err ? err.stdout : '') || '';
-      if (!stdout.trim()) throw err;
-    }
-    return { runId: manifest.run_id, ...JSON.parse(stdout) };
-  } finally {
-    await rm(dir, { recursive: true, force: true });
-  }
-}
-
-const rows = [];
-let failures = 0;
-
-for (const entry of CORPUS) {
-  process.stdout.write(`── ${entry.id}: `);
-  let result;
-  try {
-    result = await replay(entry);
-  } catch (err) {
-    failures += 1;
-    console.log('replay did not complete');
-    console.error(`     ${String(err).split('\n')[0]}`);
-    rows.push({ entry, broken: true });
-    continue;
-  }
-
-  // Field names come from `replay --json`, read off a real run rather than assumed.
-  const {
-    unmatched = 0,
-    canonicalExact,
-    canonical_exact: legacyExact = 0,
-    divergences = 0,
-    liveCalls = 0,
-  } = result;
-  const exact = canonicalExact ?? legacyExact;
-  const problems = [];
-  if (unmatched > 0) problems.push(`${unmatched} unmatched (budget 0)`);
-  if (exact < entry.budget.minExact) {
-    problems.push(`${exact} canonical_exact, expected at least ${entry.budget.minExact}`);
-  }
-  if (divergences > entry.budget.maxDivergences) {
-    problems.push(`${divergences} divergences, budget ${entry.budget.maxDivergences}`);
-  }
-
-  // A replay that went live answered a turn from a model, which is the one thing an offline
-  // guarantee cannot survive quietly.
-  if (liveCalls > 0) problems.push(`${liveCalls} live call(s) — the replay left the machine`);
-
-  if (problems.length > 0) {
-    failures += 1;
-    console.log('regressed');
-    for (const p of problems) console.error(`     ${p}`);
-  } else {
-    console.log(`canonical_exact ${exact}, unmatched ${unmatched}, divergences ${divergences}`);
-  }
-  rows.push({ entry, unmatched, exact, divergences, liveCalls, problems });
-}
-
-if (!process.argv.includes('--check')) {
-  const lines = [
-    '# Replay fidelity',
-    '',
-    '<!-- Generated by scripts/fidelity.mjs. Do not edit by hand. -->',
-    '',
-    'Every recording below is replayed with the network off and the numbers asserted. A turn the',
-    'recording cannot serve — `unmatched` — has a budget of zero, because a replay that has to go',
-    'and ask a model is not a replay. `canonical_exact` counts turns matched after canonical normalization; a floor is',
-    'set per entry so fidelity carried by the match ladder rather than by the capture shows up as',
-    'a regression rather than as a pass.',
-    '',
-    '| Harness | Canonical exact | Unmatched | Divergences | Live calls | Budget |',
-    '|---|---|---|---|---|---|',
-  ];
-  for (const r of rows) {
-    if (r.broken) {
-      lines.push(`| ${r.entry.harness} | — | — | — | — | replay did not complete |`);
-      continue;
-    }
-    lines.push(
-      `| ${r.entry.harness} | ${r.exact} | ${r.unmatched} | ${r.divergences} | ${r.liveCalls} | ` +
-        `canonical exact ≥ ${r.entry.budget.minExact}, divergences ≤ ${r.entry.budget.maxDivergences} |`,
-    );
-  }
-  lines.push('');
-  for (const r of rows) lines.push(`- **${r.entry.harness}** — ${r.entry.note}`);
-  lines.push('');
-  await writeFile(join(root, 'docs', 'fidelity.md'), `${lines.join('\n')}\n`, 'utf8');
-  console.log('\ndocs/fidelity.md written');
-}
-
-console.log(`\n${CORPUS.length} recording(s) replayed, ${failures} regression(s)`);
-process.exit(failures === 0 ? 0 : 1);

package/scripts/hermes-client-acceptance.mjs

@@ -1,158 +0,0 @@
-// Native Hermes TLS acceptance. Set AGENTICREPLAY_TEST_HERMES to the isolated executable.
-// Uses synthetic credentials and a loopback origin; nip.io DNS requires network access.
-import assert from 'node:assert/strict';
-import { createServer } from 'node:https';
-import { mkdtemp, mkdir, readFile, writeFile } from 'node:fs/promises';
-import { spawn, execFile } from 'node:child_process';
-import { promisify } from 'node:util';
-import { dirname, join } from 'node:path';
-import { pathToFileURL } from 'node:url';
-const root = await mkdtemp('/private/tmp/ar-hermes-native-');
-const home = join(root, 'home');
-await mkdir(home);
-const hermes = process.env.AGENTICREPLAY_TEST_HERMES;
-assert.ok(hermes, 'AGENTICREPLAY_TEST_HERMES must name an isolated Hermes installation');
-const host = '127.0.0.1.nip.io';
-const cert = join(root, 'cert.pem');
-const key = join(root, 'key.pem');
-await promisify(execFile)('openssl', [
-  'req',
-  '-x509',
-  '-newkey',
-  'rsa:2048',
-  '-nodes',
-  '-keyout',
-  key,
-  '-out',
-  cert,
-  '-days',
-  '1',
-  '-subj',
-  `/CN=${host}`,
-  '-addext',
-  `subjectAltName=DNS:${host}`,
-]);
-const calls = [];
-const stream = `data: ${JSON.stringify({ id: 'chatcmpl-fixture', object: 'chat.completion.chunk', model: 'stub-model', choices: [{ index: 0, delta: { role: 'assistant', content: 'HERMES_TRAILING_SENTINEL' }, finish_reason: null }] })}\n\ndata: ${JSON.stringify({ id: 'chatcmpl-fixture', object: 'chat.completion.chunk', model: 'stub-model', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], usage: { prompt_tokens: 12, completion_tokens: 4, total_tokens: 16 } })}\n\ndata: [DONE]\n\n`;
-const server = createServer(
-  { key: await readFile(key), cert: await readFile(cert) },
-  async (req, res) => {
-    let body = '';
-    for await (const chunk of req) body += chunk;
-    const response =
-      req.url === '/v1/models'
-        ? JSON.stringify({
-            object: 'list',
-            data: [{ id: 'stub-model', object: 'model', owned_by: 'fixture' }],
-          })
-        : stream;
-    calls.push({ path: req.url, body, response });
-    res.writeHead(200, {
-      'content-type': req.url === '/v1/models' ? 'application/json' : 'text/event-stream',
-    });
-    res.write(response.slice(0, -12));
-    setTimeout(() => res.end(response.slice(-12)), 10);
-  },
-);
-await new Promise((r) => server.listen(0, '127.0.0.1', r));
-await writeFile(
-  join(home, 'config.yaml'),
-  `_config_version: 12\nmodel:\n  provider: custom\n  default: stub-model\n  base_url: https://${host}:${server.address().port}/v1\n  api_key: fixture-key\n  context_length: 64000\n`,
-);
-const cli = process.env.AGENTICREPLAY_TEST_CLI ?? join(process.cwd(), 'packages/cli/dist/cli.js');
-const env = {
-  PATH: `${dirname(process.execPath)}:${dirname(hermes)}:/opt/homebrew/bin:/usr/bin:/bin`,
-  HOME: home,
-  HERMES_HOME: home,
-  XDG_CONFIG_HOME: join(root, 'config'),
-  XDG_DATA_HOME: join(root, 'data'),
-  XDG_CACHE_HOME: join(root, 'cache'),
-  XDG_STATE_HOME: join(root, 'state'),
-  PYTHONNOUSERSITE: '1',
-  PYTHONUNBUFFERED: '1',
-  NODE_EXTRA_CA_CERTS: cert,
-  OPENAI_API_KEY: 'fixture-key',
-};
-const child = spawn(
-  process.execPath,
-  [
-    cli,
-    'record',
-    'hermes',
-    '--tls-intercept',
-    '--tls-hosts',
-    `+${host}`,
-    '--no-agent-spans',
-    '--no-fs',
-    '--no-shell',
-    '--json',
-    '--',
-    '--provider',
-    'custom',
-    '--model',
-    'stub-model',
-    '--no-restore-cwd',
-    '-z',
-    'HERMES_SYSTEM_REQUEST_SENTINEL',
-  ],
-  { cwd: root, env, detached: true, stdio: ['ignore', 'pipe', 'pipe'] },
-);
-let stdout = '',
-  stderr = '';
-child.stdout.on('data', (c) => (stdout += c));
-child.stderr.on('data', (c) => (stderr += c));
-const timer = setTimeout(() => {
-  try {
-    process.kill(-child.pid, 'SIGKILL');
-  } catch {}
-}, 45000);
-const outcome = await new Promise((r) => child.on('close', (code, signal) => r({ code, signal })));
-clearTimeout(timer);
-server.closeAllConnections();
-server.close();
-assert.ok([0, 2].includes(outcome.code), stderr);
-assert.ok(stderr.includes('HERMES_TRAILING_SENTINEL'), stderr);
-const run = JSON.parse(stdout.trim());
-assert.equal(run.childExitCode, 0);
-const { TraceReader } = await import(
-  pathToFileURL(
-    process.env.AGENTICREPLAY_TEST_TRACE_READER ??
-      join(process.cwd(), 'packages/core/dist/index.js'),
-  ).href
-);
-const reader = await TraceReader.open(run.runDir);
-const events = await reader.events();
-const requests = events.filter((e) => e.type === 'net.request' && e.attrs.phase === 'body');
-const responses = events.filter((e) => e.type === 'net.response');
-assert.equal(requests.length, calls.length, 'missing transport evidence');
-assert.equal(responses.length, calls.length);
-const remaining = [...calls];
-for (const request of requests) {
-  const body = await reader.resolvePayload(request);
-  const index = remaining.findIndex((c) => c.body === body && c.path === request.attrs.path);
-  assert.ok(index >= 0, 'request differs from independent endpoint');
-  const [expected] = remaining.splice(index, 1);
-  const response = responses.find((e) => e.attrs.request_id === request.attrs.request_id);
-  assert.equal(await reader.resolvePayload(response), expected.response);
-  assert.equal(response.attrs.capture_incomplete, false);
-}
-assert.ok(
-  calls.some(
-    (c) =>
-      c.body.includes('HERMES_SYSTEM_REQUEST_SENTINEL') && JSON.parse(c.body).tools?.length > 0,
-  ),
-);
-const manifest = JSON.parse(await readFile(join(run.runDir, 'manifest.json'), 'utf8'));
-assert.equal(outcome.code, manifest.capture.status === 'complete' ? 0 : 2);
-console.log(
-  JSON.stringify({
-    root,
-    ...outcome,
-    requests: calls.length,
-    exactBodies: true,
-    system: true,
-    tools: true,
-    capture: manifest.capture,
-    runDir: run.runDir,
-  }),
-);