agenticreplay 0.1.0 → 0.1.2

package/packages/proxy/test/indexing-concurrency.test.ts

@@ -1,224 +0,0 @@
-import { describe, expect, it } from 'vitest';
-import type { CanonicalRequest } from '@agenticreplay/plugin-api';
-import { RequestMatcher } from '../src/matching.js';
-
-/**
- * The indexing workload: many requests that share an instruction and differ only in the document.
- *
- * This is the shape every index-building RAG pipeline has — IndexRAG, Microsoft GraphRAG,
- * LightRAG, LlamaIndex ingestion — and it was the one shape the ladder got wrong. The prompt
- * template is the *message*; the paragraph being processed rides in the system prompt. So every
- * request's trailing message is identical to every other's, `sameAsk` is true against all of
- * them, and rung 3 — whose stated reason for existing is context compaction, which rewrites
- * `messages` — waved each one through against whichever recorded request the cursor was on.
- *
- * Measured on 30 real HotpotQA paragraphs at concurrency 10 before the guard: 24 requests served
- * another paragraph's answer, `divergences=24`, `unmatched=0`, `exit=0`. The knowledge base built
- * from a replay of that recording is a knowledge base of mismatched answers, and nothing in the
- * run says so.
- */
-function extraction(i: number): CanonicalRequest {
-  return {
-    model: 'gpt-4o-mini',
-    // The paragraph. Different every time, and not in `messages`.
-    system: `Extract atomic knowledge units.\n\nDocument ${i}: ${'lorem ipsum '.repeat(20)}${i}`,
-    // The instruction. Byte-identical across all 30 requests.
-    messages: [{ role: 'user', content: [{ type: 'text', text: 'Return the AKUs as JSON.' }] }],
-    temperature: 0,
-  };
-}
-
-/** The order a worker pool of `width` hands work back: reversed inside each window. */
-function shuffled(total: number, width: number): number[] {
-  const out: number[] = [];
-  for (let start = 0; start < total; start += width) {
-    const window = [];
-    for (let i = start; i < Math.min(start + width, total); i += 1) window.push(i);
-    out.push(...window.reverse());
-  }
-  return out;
-}
-
-describe('a concurrent indexing pipeline', () => {
-  it('serves each request its own recorded answer, out of order, at width 10', () => {
-    const recorded = Array.from({ length: 30 }, (_, i) => extraction(i));
-    const matcher = new RequestMatcher(recorded, { concurrency: () => 10 });
-
-    for (const i of shuffled(30, 10)) {
-      const result = matcher.match(extraction(i));
-      expect(result.matched, `request ${i} did not match`).toBe(true);
-      expect(result.index, `request ${i} was served request ${result.index}'s answer`).toBe(i);
-    }
-  });
-
-  /**
-   * The window is a floor, not a ceiling, and a byte-identical match is not bounded by it at all.
-   * A pipeline whose requests came back in a wholly different order still gets each one its own
-   * answer — the reach is safe precisely because "identical after redaction" cannot be a
-   * coincidence.
-   */
-  it('serves them in fully reversed order, past any fixed window', () => {
-    const recorded = Array.from({ length: 30 }, (_, i) => extraction(i));
-    const matcher = new RequestMatcher(recorded);
-
-    for (let i = 29; i >= 0; i -= 1) {
-      const result = matcher.match(extraction(i));
-      expect(result.matched, `request ${i} did not match`).toBe(true);
-      expect(result.index).toBe(i);
-    }
-    expect(matcher.remaining()).toBe(0);
-  });
-
-  /**
-   * The one that survived the rung-3 guard and was found by a real replay.
-   *
-   * Rung 2 measures drift as a share of the request's own size, and an extraction prompt is
-   * mostly a fixed template — so two paragraphs differing by a few hundred characters inside a
-   * request of several thousand fall *inside* tolerance, `sameAsk` is true because the only
-   * message is a constant instruction, and the cursor's near-match was taken while the real
-   * answer sat unplayed further along. Measured on three real documents: `exact=1`, five `minor`
-   * divergences, and a cache whose chunks carried each other's extractions.
-   *
-   * The fix is an ordering rather than another guard: an exact match anywhere outranks an
-   * approximate one at the cursor.
-   */
-  it('prefers its own answer further along over a near-match at the cursor', () => {
-    // Close enough to be inside rung 2's tolerance against each other, and not identical.
-    const template = 'Extract the facts.\n'.repeat(60);
-    const near = (i: number): CanonicalRequest => ({
-      model: 'gpt-4o-mini',
-      system: `${template}Document: ${'x'.repeat(200)}${i}`,
-      messages: [{ role: 'user', content: [{ type: 'text', text: 'Return JSON.' }] }],
-    });
-
-    // Proof the trap is real: against request 0, request 1 is a rung-2 match.
-    const trap = new RequestMatcher([near(0)]).match(near(1));
-    expect(trap.matched).toBe(true);
-    expect(trap.rung).toBe(2);
-
-    // And with its own answer in the recording, that is what it gets.
-    const matcher = new RequestMatcher([near(0), near(1), near(2)]);
-    expect(matcher.match(near(1)).index).toBe(1);
-    expect(matcher.match(near(2)).index).toBe(2);
-    expect(matcher.match(near(0)).index).toBe(0);
-  });
-
-  /**
-   * The residue, once nothing in the recording is byte-identical any more.
-   *
-   * A real recording carries per-run values — a session id, a timestamp — so a replay's request
-   * is rarely byte-equal even when it is unmistakably the same question, and the exact path above
-   * does not fire. The ladder then has to approximate, and *which* recorded request it
-   * approximates against is the whole answer: on eight real documents at concurrency 10 the
-   * approximations still landed on the wrong document, because the cursor's near-match was taken
-   * for being the cursor's rather than for being near.
-   *
-   * `leafDistance` already knows which candidate is closer. Using it is the fix.
-   */
-  it('approximates against the nearest recorded request, not the nearest to hand', () => {
-    const template = 'Extract the facts.\n'.repeat(60);
-    // The drifting part: present in every request, different on every run, so nothing matches
-    // exactly and every comparison lands on rung 2.
-    const drifted = (i: number, run: string): CanonicalRequest => ({
-      model: 'gpt-4o-mini',
-      system: `${template}Session ${run}\nDocument: ${'x'.repeat(200)}${i}`,
-      messages: [{ role: 'user', content: [{ type: 'text', text: 'Return JSON.' }] }],
-    });
-
-    // Proof the trap is real: on its own, document 0 *does* accept document 2's request at
-    // rung 2 — which is what the cursor would have served.
-    const trap = new RequestMatcher([drifted(0, 'recorded')]).match(drifted(2, 'replayed'));
-    expect(trap.matched).toBe(true);
-    expect(trap.rung).toBe(2);
-
-    const matcher = new RequestMatcher([0, 1, 2, 3].map((i) => drifted(i, 'recorded')));
-    // Nothing is exact, and document 2's request is a rung-2 match against document 0 sitting at
-    // the cursor. Its own answer is two positions along and closer.
-    expect(matcher.match(drifted(2, 'replayed')).index).toBe(2);
-    expect(matcher.match(drifted(0, 'replayed')).index).toBe(0);
-    expect(matcher.match(drifted(3, 'replayed')).index).toBe(3);
-    expect(matcher.match(drifted(1, 'replayed')).index).toBe(1);
-  });
-
-  /**
-   * The same ordering question, in the state a worker pool reaches most easily of all.
-   *
-   * A pool that hands its last document back first drives the cursor past the end of the
-   * recording on the *first* request, and every request after that is answered out of the held
-   * set alone. That branch served whichever held request matched first in insertion order — the
-   * earliest document, not the nearest one — so the answers came back shuffled: on four documents
-   * arriving 3, 2, 1, 0, documents 2 and 0 were each served the other's extraction, and at width
-   * 10 over thirty documents eight of them were. `unmatched=0`, `exit=0`, every mis-serve a
-   * `minor` divergence, and a knowledge base built from the replay that is quietly wrong.
-   *
-   * Two held candidates are the minimum that can tell the two orderings apart, which is why
-   * nothing caught this: the recording exhaustion test holds none, and the lookahead test holds
-   * one, and with one candidate "first" and "nearest" are the same request.
-   */
-  it('ranks the held requests when the cursor has run off the end', () => {
-    const template = 'Extract the facts.\n'.repeat(60);
-    const drifted = (i: number, run: string): CanonicalRequest => ({
-      model: 'gpt-4o-mini',
-      system: `${template}Session ${run}\nDocument: ${'x'.repeat(200)}${i}`,
-      messages: [{ role: 'user', content: [{ type: 'text', text: 'Return JSON.' }] }],
-    });
-
-    const matcher = new RequestMatcher(
-      [0, 1, 2, 3].map((i) => drifted(i, 'recorded')),
-      { concurrency: () => 4 },
-    );
-
-    // Document 3 first: its own answer is the last one recorded, so the cursor lands past the end
-    // and 0, 1 and 2 are all held.
-    expect(matcher.match(drifted(3, 'replayed')).index).toBe(3);
-    expect(matcher.remaining()).toBe(3);
-
-    // From here every answer comes out of the held set, and each has to be the right one. Document
-    // 2 is where it used to go wrong: index 0 is first in the held set and matches at rung 2.
-    expect(matcher.match(drifted(2, 'replayed')).index).toBe(2);
-    expect(matcher.match(drifted(1, 'replayed')).index).toBe(1);
-    expect(matcher.match(drifted(0, 'replayed')).index).toBe(0);
-    expect(matcher.remaining()).toBe(0);
-  });
-
-  /**
-   * The guard itself, isolated. Same ask, same message count, a system prompt that differs by more
-   * than rung 2's tolerance: that is a different request, and the honest answer is rung 4.
-   */
-  it('refuses a different document at the cursor rather than calling it a compaction', () => {
-    const matcher = new RequestMatcher([extraction(0)]);
-    const result = matcher.match(extraction(1));
-    expect(result.matched).toBe(false);
-    expect(result.rung).toBe(4);
-  });
-
-  /**
-   * What rung 3 is actually for, unchanged. Compaction rewrites the history and leaves the
-   * question standing, so the message counts differ — which is the condition the guard added.
-   */
-  it('still matches a compacted conversation at rung 3', () => {
-    const ask = { role: 'user' as const, content: [{ type: 'text' as const, text: 'and now?' }] };
-    const long: CanonicalRequest = {
-      model: 'claude-opus-5',
-      messages: [
-        { role: 'user', content: [{ type: 'text', text: 'the first turn' }] },
-        { role: 'assistant', content: [{ type: 'text', text: 'an answer' }] },
-        { role: 'user', content: [{ type: 'text', text: 'a second turn' }] },
-        { role: 'assistant', content: [{ type: 'text', text: 'another answer' }] },
-        ask,
-      ],
-    };
-    const compacted: CanonicalRequest = {
-      model: 'claude-opus-5',
-      messages: [
-        { role: 'user', content: [{ type: 'text', text: 'summary of the conversation so far' }] },
-        ask,
-      ],
-    };
-
-    const result = new RequestMatcher([long]).match(compacted);
-    expect(result.matched).toBe(true);
-    expect(result.rung).toBe(3);
-    expect(result.divergence?.level).toBe('major');
-  });
-});

package/packages/proxy/test/lookahead.test.ts

@@ -1,160 +0,0 @@
-import { describe, expect, it } from 'vitest';
-import type { CanonicalRequest } from '@agenticreplay/plugin-api';
-import { RequestMatcher } from '../src/matching.js';
-
-/**
- * Stepping over what the replay did not repeat.
- *
- * A recording made through a terminal holds calls the harness made for itself: a quota probe
- * before the first turn, a request to name the session. A replay driven from a transcript never
- * makes them, and the cursor used to stop dead on the first one — so a recording of an ordinary
- * interactive session could not be replayed at all, however faithful the rest of it was.
- */
-function req(text: string, over: Partial<CanonicalRequest> = {}): CanonicalRequest {
-  return {
-    model: 'claude-opus-5',
-    messages: [{ role: 'user', content: [{ type: 'text', text }] }],
-    ...over,
-  };
-}
-
-describe('RequestMatcher lookahead', () => {
-  it('steps over a recorded request the replay did not make', () => {
-    const m = new RequestMatcher([req('quota'), req('the real question')]);
-    const result = m.match(req('the real question'));
-    expect(result.matched).toBe(true);
-    expect(result.index).toBe(1);
-    expect(result.skipped).toBe(1);
-  });
-
-  /**
-   * An exchange stepped over without mention is a replay claiming to reproduce what it passed by
-   * — so it is reported. Not as a divergence, though: spec §4 reserves those for a match below
-   * rung 1, and this one is byte-identical after redaction. Calling a reordering a divergence
-   * made `exact=N/N` unreachable for any concurrent workload, and what was actually never
-   * reproduced is still counted by `reused=x/y`.
-   */
-  it('says it stepped over, without calling an exact match a divergence', () => {
-    const m = new RequestMatcher([req('quota'), req('ask')]);
-    const result = m.match(req('ask'));
-    expect(result.rung).toBe(1);
-    expect(result.skipped).toBe(1);
-    expect(result.reordered).toBe(true);
-    expect(result.divergence).toBeUndefined();
-  });
-
-  // A skip onto an *inexact* match keeps its divergence, and names the skip inside it.
-  it('still reports a divergence when the match it stepped onto was approximate', () => {
-    const drifted = req('a question about the quarterly billing report for account 41');
-    const m = new RequestMatcher([req('quota'), drifted]);
-    const result = m.match(req('a question about the quarterly billing report for account 42'));
-    expect(result.matched).toBe(true);
-    expect(result.rung).toBe(2);
-    expect(result.divergence?.detail ?? '').toContain('holding 1 recorded request');
-  });
-
-  /**
-   * The case that made stepping over reversible.
-   *
-   * goose asks a second model for a session title while the conversation is already under way, so
-   * the title call and the first real turn race and do not land in the same order twice. Stepping
-   * over used to *discard* what it passed, which meant a replay whose title arrived first threw
-   * away the first turn on its way to matching the title — and then halted on the very next
-   * request, against a recording that contained everything it needed.
-   */
-  it('still serves a request it stepped over, when that is what comes next', () => {
-    const m = new RequestMatcher([req('the first turn'), req('name this session')]);
-
-    const title = m.match(req('name this session'));
-    expect(title.index).toBe(1);
-    expect(title.skipped).toBe(1);
-
-    const turn = m.match(req('the first turn'));
-    expect(turn.matched).toBe(true);
-    expect(turn.index).toBe(0);
-    // Reported as a reordering rather than a divergence: both halves of the pair were reproduced
-    // byte for byte, and the only thing that differed was which arrived first.
-    expect(turn.reordered).toBe(true);
-    expect(turn.divergence).toBeUndefined();
-  });
-
-  it('serves a held request even after the cursor has run off the end', () => {
-    const m = new RequestMatcher([req('held back'), req('last')]);
-    expect(m.match(req('last')).index).toBe(1);
-    expect(m.match(req('held back')).index).toBe(0);
-  });
-
-  // Held, not forgotten: a request set aside is still owed to the replay, and `reused=x/y` counts
-  // it as unplayed until it is served.
-  it('counts a held request as still remaining', () => {
-    const m = new RequestMatcher([req('held back'), req('asked first')]);
-    m.match(req('asked first'));
-    expect(m.remaining()).toBe(1);
-    m.match(req('held back'));
-    expect(m.remaining()).toBe(0);
-  });
-
-  // The same bar as a step-forward. Below rung 2 the agent would be handed some other turn's
-  // answer, which is worse than halting and saying so.
-  it('does not serve a held request on a weak match', () => {
-    const m = new RequestMatcher([req('a question about billing'), req('asked first')]);
-    m.match(req('asked first'));
-    const result = m.match(req('an unrelated question about deployment'));
-    expect(result.matched).toBe(false);
-    expect(result.rung).toBe(4);
-  });
-
-  it('does not step over anything when the cursor already matches', () => {
-    const m = new RequestMatcher([req('a'), req('b')]);
-    const result = m.match(req('a'));
-    expect(result.index).toBe(0);
-    expect(result.skipped).toBeUndefined();
-  });
-
-  /**
-   * Only an exact or near-exact match is worth stepping over an unplayed exchange for. A loose one
-   * would hand the agent some other turn's answer, which is worse than halting and saying so.
-   */
-  it('halts rather than stepping onto a weak match', () => {
-    const m = new RequestMatcher([req('first question'), req('a completely different question')]);
-    const result = m.match(req('something else again'));
-    expect(result.matched).toBe(false);
-    expect(result.rung).toBe(4);
-  });
-
-  it('leaves the cursor past the request it matched, not past the one it skipped', () => {
-    const m = new RequestMatcher([req('quota'), req('one'), req('two')]);
-    m.match(req('one'));
-    expect(m.match(req('two')).index).toBe(2);
-  });
-
-  /**
-   * A session recorded through a terminal is offered tools that need one — asking a question,
-   * entering plan mode. Their schemas are large, so the distance runs into six figures and reads
-   * as a corrupted trace rather than as the same agent started two different ways.
-   */
-  it('names the tools the recording had that the replay cannot', () => {
-    const withTools = req('ask', {
-      tools: [
-        { name: 'Read', description: '', input_schema: {} },
-        { name: 'AskUserQuestion', description: '', input_schema: {} },
-      ],
-    });
-    const withoutTools = req('a different ask entirely', {
-      tools: [{ name: 'Read', description: '', input_schema: {} }],
-    });
-    const result = new RequestMatcher([withTools]).match(withoutTools);
-    expect(result.matched).toBe(false);
-    expect(result.reason).toContain('AskUserQuestion');
-    expect(result.reason).toContain('a session recorded through a terminal');
-  });
-
-  it('says nothing about tools when both sides offer the same ones', () => {
-    const tools = [{ name: 'Read', description: '', input_schema: {} }];
-    const result = new RequestMatcher([req('one thing', { tools })]).match(
-      req('a totally unrelated other thing', { tools }),
-    );
-    expect(result.matched).toBe(false);
-    expect(result.reason).not.toContain('tools');
-  });
-});

package/packages/proxy/test/malformed-body.test.ts

@@ -1,73 +0,0 @@
-import { afterEach, describe, expect, it } from 'vitest';
-import { createProxy } from '../src/server.js';
-
-const closers: Array<() => Promise<void>> = [];
-afterEach(async () => {
-  while (closers.length) await closers.pop()!();
-});
-
-/**
- * A request body the proxy cannot parse.
- *
- * Record mode parsed the body unguarded, so anything malformed reached the server's catch-all and
- * came back as `500 {"error":{"message":"SyntaxError: Unexpected end of JSON input"}}`. To whoever
- * is running the agent that reads as agenticreplay falling over, and it sends them to agenticreplay's issue tracker
- * with a stack trace instead of to the one line of their own harness that sent an empty body.
- * Replay already answered 400 with a real message; record should too.
- */
-const upstream = (async () =>
-  new Response('{}', {
-    status: 200,
-    headers: { 'content-type': 'application/json' },
-  })) as unknown as typeof fetch;
-
-async function post(url: string, body: string) {
-  const res = await fetch(url, {
-    method: 'POST',
-    headers: { 'content-type': 'application/json' },
-    body,
-  });
-  return { status: res.status, text: await res.text() };
-}
-
-describe('record mode — a body the dialect cannot read', () => {
-  for (const [label, body] of [
-    ['an empty body', ''],
-    ['truncated JSON', '{"model":"gpt-5.2","inp'],
-    ['not JSON at all', 'hello'],
-  ] as const) {
-    it(`answers 400 rather than 500 for ${label}`, async () => {
-      const proxy = await createProxy({ mode: 'record', fetchImpl: upstream });
-      closers.push(proxy.close);
-
-      const res = await post(`${proxy.url}/v1/responses`, body);
-
-      expect(res.status).toBe(400);
-      // Name the path, so the operator can find the call in their own code.
-      expect(res.text).toContain('/v1/responses');
-    });
-  }
-
-  it('does not forward an unparseable body upstream', async () => {
-    let called = 0;
-    const counting = (async () => {
-      called += 1;
-      return new Response('{}', { status: 200 });
-    }) as unknown as typeof fetch;
-    const proxy = await createProxy({ mode: 'record', fetchImpl: counting });
-    closers.push(proxy.close);
-
-    await post(`${proxy.url}/v1/responses`, '');
-
-    expect(called).toBe(0);
-  });
-
-  it('records nothing for a request it could not read', async () => {
-    const proxy = await createProxy({ mode: 'record', fetchImpl: upstream });
-    closers.push(proxy.close);
-
-    await post(`${proxy.url}/v1/responses`, '');
-
-    expect(proxy.exchanges()).toHaveLength(0);
-  });
-});