agenticreplay 0.1.0 → 0.1.2

package/packages/cli/test/attach-command.test.ts

@@ -1,356 +0,0 @@
-import { execFile } from 'node:child_process';
-import { mkdtemp, readdir, rm } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { promisify } from 'node:util';
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-import { TraceReader } from '@agenticreplay/core';
-import { parseArgs } from '../src/args.js';
-import { Output } from '../src/out.js';
-import { attachCommand } from '../src/commands/attach.js';
-import { replayCommand } from '../src/commands/replay.js';
-import { startFakeModel } from './fixtures/fake-model.mjs';
-
-const run = promisify(execFile);
-
-/**
- * `agenticreplay attach` — a recording session for an agent agenticreplay did not start.
- *
- * The command holds a proxy open, prints what to export, and records whatever arrives until it is
- * told to stop. That makes it the answer for an agent in a sandbox, on a VPS, or in someone's CI:
- * places where there is no process for agenticreplay to spawn and therefore no environment for it to build.
- *
- * These tests stand in for the sandbox with an ordinary `fetch` from this process. The distance
- * does not change anything that could break — what matters is that the run is driven entirely from
- * the outside, and that the trace it leaves behind replays like any other.
- */
-describe('agenticreplay attach', () => {
-  let workspace: string;
-  let out: Output;
-  let lines: string[];
-  let model: Awaited<ReturnType<typeof startFakeModel>>;
-
-  beforeEach(async () => {
-    workspace = await mkdtemp(join(tmpdir(), 'agenticreplay-attach-'));
-    model = await startFakeModel();
-    lines = [];
-    out = new Output({ write: (s) => void lines.push(s), isTTY: false });
-    await run('git', ['init', '-q'], { cwd: workspace });
-    await run('git', ['config', 'user.email', 'test@example.com'], { cwd: workspace });
-    await run('git', ['config', 'user.name', 'Test'], { cwd: workspace });
-  });
-  afterEach(async () => {
-    await model.close();
-    await rm(workspace, { recursive: true, force: true });
-  });
-
-  /** Runs an attach session, handing the caller the live proxy url to drive it with. */
-  function session(
-    argv: string[],
-    drive: (proxyUrl: string) => Promise<void>,
-  ): ReturnType<typeof attachCommand> {
-    let release: () => void = () => {};
-    const until = new Promise<void>((resolve) => (release = resolve));
-    return attachCommand(
-      parseArgs(['attach', '--upstream-anthropic', model.url, ...argv]),
-      out,
-      workspace,
-      {
-        until,
-        onReady: (info) => {
-          void drive(info.proxyUrl).finally(release);
-        },
-      },
-    );
-  }
-
-  it('prints the block to paste, pointed at the address it is reachable on', async () => {
-    await session(['--for', 'claude'], async () => {});
-    const printed = lines.join('');
-    expect(printed).toContain('export ANTHROPIC_BASE_URL=');
-    expect(printed).toMatch(/http:\/\/127\.0\.0\.1:\d+/);
-  });
-
-  it('refuses a wildcard bind with no advertised host, rather than printing an unusable url', async () => {
-    await expect(session(['--bind', '0.0.0.0'], async () => {})).rejects.toThrow(/--advertise/);
-  });
-
-  /**
-   * The refusal has to happen before anything is created, not merely before anything is printed.
-   * Validating only in the default-port path meant `--port` skipped the check entirely: the run
-   * directory, the trace and (with --tls-intercept) a private key were all created for a session
-   * that was about to throw.
-   */
-  it('refuses a wildcard bind before it creates a run, whatever the port', async () => {
-    await expect(session(['--bind', '0.0.0.0', '--port', '18771'], async () => {})).rejects.toThrow(
-      /--advertise/,
-    );
-    const runs = await readdir(join(workspace, '.agenticreplay', 'runs')).catch(() => []);
-    expect(runs).toEqual([]);
-  });
-
-  /**
-   * A session nobody can connect to is the failure this command exists to prevent, and the empty
-   * block is the one form of it agenticreplay can see coming: `exec` sets no variables, so with no
-   * interception either there is literally nothing for the operator to paste.
-   */
-  it('says so when there is nothing to paste, instead of printing an empty block', async () => {
-    await session([], async () => {});
-    expect(lines.join('')).toContain('attach.nothing_to_set');
-  });
-
-  /**
-   * Some adapters instrument by writing a file into the run directory and naming it in the
-   * environment — the fetch hook is the live example. Those paths exist on this machine and not in
-   * the sandbox, so printing them yields a block that runs cleanly and instruments nothing.
-   */
-  it('flags an instruction that names a file only this machine has', async () => {
-    await session(['--for', 'node'], async () => {});
-    expect(lines.join('')).toContain('attach.local_path');
-  });
-
-  it('never prints the operator’s own credential', async () => {
-    const before = process.env.ANTHROPIC_API_KEY;
-    process.env.ANTHROPIC_API_KEY = 'sk-ant-REAL-SECRET';
-    try {
-      await session(['--for', 'claude'], async () => {});
-    } finally {
-      if (before === undefined) delete process.env.ANTHROPIC_API_KEY;
-      else process.env.ANTHROPIC_API_KEY = before;
-    }
-    expect(lines.join('')).not.toContain('sk-ant-REAL-SECRET');
-  });
-
-  /**
-   * The result is what `--json` prints and what a script reads. Returning the bind address there
-   * hands a caller the exact unusable URL `advertisedUrl` throws to prevent.
-   */
-  it('reports the address the agent should use, not the one it bound', async () => {
-    const result = await session(
-      ['--for', 'claude', '--bind', '0.0.0.0', '--advertise', 'host.docker.internal'],
-      async () => {},
-    );
-    expect(result.proxyUrl).toMatch(/^http:\/\/host\.docker\.internal:\d+$/);
-  });
-
-  /**
-   * `--replay` takes a run, and the parser turns a valueless flag into `true` — so a bare
-   * `agenticreplay attach --replay`, or `--replay --tls-intercept`, silently became a fresh *recording*
-   * session pointed at the live provider. Exactly the wrong direction to fail in.
-   */
-  it('refuses --replay with no run rather than recording instead', async () => {
-    await expect(session(['--replay'], async () => {})).rejects.toThrow(/--replay/);
-  });
-
-  it('leaves no half-made run behind when the session cannot start', async () => {
-    await expect(
-      session(['--tls-intercept', '--tls-hosts', 'api.openai.com,+grok.example'], async () => {}),
-    ).rejects.toThrow(/\+/);
-    const runs = await readdir(join(workspace, '.agenticreplay', 'runs')).catch(() => []);
-    expect(runs).toEqual([]);
-  });
-
-  it('advertises the host the operator names, not the one it bound', async () => {
-    await session(
-      ['--for', 'claude', '--bind', '0.0.0.0', '--advertise', 'host.docker.internal'],
-      async () => {},
-    );
-    expect(lines.join('')).toContain('http://host.docker.internal:');
-  });
-
-  it('records what an agent it never launched sends, and writes a trace', async () => {
-    const result = await session(['--for', 'claude'], async (proxyUrl) => {
-      await fetch(`${proxyUrl}/v1/messages`, {
-        method: 'POST',
-        headers: { 'content-type': 'application/json' },
-        body: JSON.stringify({
-          model: 'claude-opus-5',
-          messages: [{ role: 'user', content: 'hello from the sandbox' }],
-        }),
-      });
-    });
-
-    expect(result.modelExchanges).toBe(1);
-    const events = await (await TraceReader.open(result.runDir)).events();
-    expect(events.filter((e) => e.type === 'model.request')).toHaveLength(1);
-
-    // Spec §2.3: `run.start` and `run.end` bracket the run, exactly one of each. This assertion
-    // was the whole reason `attach` produced traces no gateway would take — it wrote neither, and
-    // the only thing anyone checked here was the model exchange.
-    expect(events[0]?.type, `first event was ${events[0]?.type}`).toBe('run.start');
-    expect(events[0]?.seq).toBe(0);
-    expect(events.at(-1)?.type, `last event was ${events.at(-1)?.type}`).toBe('run.end');
-  });
-
-  /**
-   * Replaying a run whose agent is somewhere else.
-   *
-   * `agenticreplay replay` launches the agent, which is the right thing for a run agenticreplay recorded by
-   * launching it — and impossible for one that came from a sandbox, where the agent may not exist
-   * on this machine at all. So replay attaches too: the proxy serves the recording, the operator
-   * points the same remote agent at it, and no model is called.
-   *
-   * Asserting on `reused` rather than on `unmatched` is deliberate. A replay nothing ever connects
-   * to reports zero unmatched requests, because zero requests arrived — an assertion that passes
-   * for a session that did nothing at all.
-   */
-  it('serves a recording back to an agent it does not launch', async () => {
-    const recorded = await session(['--for', 'claude'], async (proxyUrl) => {
-      await fetch(`${proxyUrl}/v1/messages`, {
-        method: 'POST',
-        headers: { 'content-type': 'application/json' },
-        body: JSON.stringify({
-          model: 'claude-opus-5',
-          messages: [{ role: 'user', content: 'hello from the sandbox' }],
-        }),
-      });
-    });
-
-    let body = '';
-    const replayed = await session(['--replay', recorded.runId], async (proxyUrl) => {
-      const res = await fetch(`${proxyUrl}/v1/messages`, {
-        method: 'POST',
-        headers: { 'content-type': 'application/json' },
-        body: JSON.stringify({
-          model: 'claude-opus-5',
-          messages: [{ role: 'user', content: 'hello from the sandbox' }],
-        }),
-      });
-      body = await res.text();
-    });
-
-    expect(replayed.reused).toBe(1);
-    expect(replayed.unmatched).toBe(0);
-    // A replay session reports what it served, not a recording it did not make.
-    expect(lines.join('')).toContain('info served');
-    expect(lines.join('')).toContain('reused=1/1');
-    // Served from the trace, so the model this test stood up was never called a second time.
-    expect(model.calls).toHaveLength(1);
-    expect(body).toContain('editing auth.ts');
-  });
-
-  /**
-   * A replay session has to hand the operator something to paste, and `exec` — the honest default
-   * when agenticreplay does not know what will connect — sets nothing at all. For a replay it does know:
-   * the trace records which adapter made the recording, so the variables that agent reads are the
-   * variables it will read again.
-   */
-  it('takes the agent from the recording, so the block is not empty', async () => {
-    const recorded = await session(['--for', 'claude'], async (proxyUrl) => {
-      await fetch(`${proxyUrl}/v1/messages`, {
-        method: 'POST',
-        headers: { 'content-type': 'application/json' },
-        body: JSON.stringify({
-          model: 'claude-opus-5',
-          messages: [{ role: 'user', content: 'x' }],
-        }),
-      });
-    });
-    lines.length = 0;
-
-    await session(['--replay', recorded.runId], async () => {});
-
-    expect(lines.join('')).toContain('export ANTHROPIC_BASE_URL=');
-  });
-
-  it('still lets --for override the agent the recording names', async () => {
-    const recorded = await session(['--for', 'claude'], async (proxyUrl) => {
-      await fetch(`${proxyUrl}/v1/messages`, {
-        method: 'POST',
-        headers: { 'content-type': 'application/json' },
-        body: JSON.stringify({
-          model: 'claude-opus-5',
-          messages: [{ role: 'user', content: 'x' }],
-        }),
-      });
-    });
-    lines.length = 0;
-
-    await session(['--replay', recorded.runId, '--for', 'generic-openai'], async () => {});
-
-    expect(lines.join('')).toContain('export OPENAI_BASE_URL=');
-  });
-
-  /**
-   * A replay session that could not answer the agent has to be detectable by a script, not only
-   * by a human reading the log — CI is exactly where a sandbox replay runs.
-   */
-  it('reports a request the recording could not answer', async () => {
-    const recorded = await session(['--for', 'claude'], async (proxyUrl) => {
-      await fetch(`${proxyUrl}/v1/messages`, {
-        method: 'POST',
-        headers: { 'content-type': 'application/json' },
-        body: JSON.stringify({
-          model: 'claude-opus-5',
-          messages: [{ role: 'user', content: 'x' }],
-        }),
-      });
-    });
-
-    const replayed = await session(['--replay', recorded.runId], async (proxyUrl) => {
-      // A question the recording never held an answer for.
-      await fetch(`${proxyUrl}/v1/messages`, {
-        method: 'POST',
-        headers: { 'content-type': 'application/json' },
-        body: JSON.stringify({
-          model: 'claude-opus-5',
-          messages: [{ role: 'user', content: 'something else entirely' }],
-        }),
-      });
-    });
-
-    expect(replayed.unmatched).toBeGreaterThan(0);
-    expect(replayed.exitCode).toBe(1);
-  });
-
-  it('exits zero for a session that served everything it was asked for', async () => {
-    const recorded = await session(['--for', 'claude'], async (proxyUrl) => {
-      await fetch(`${proxyUrl}/v1/messages`, {
-        method: 'POST',
-        headers: { 'content-type': 'application/json' },
-        body: JSON.stringify({
-          model: 'claude-opus-5',
-          messages: [{ role: 'user', content: 'x' }],
-        }),
-      });
-    });
-    const replayed = await session(['--replay', recorded.runId], async (proxyUrl) => {
-      await fetch(`${proxyUrl}/v1/messages`, {
-        method: 'POST',
-        headers: { 'content-type': 'application/json' },
-        body: JSON.stringify({
-          model: 'claude-opus-5',
-          messages: [{ role: 'user', content: 'x' }],
-        }),
-      });
-    });
-    expect(replayed.exitCode).toBe(0);
-  });
-
-  it('reports incomplete coverage when a recording session observed no requests', async () => {
-    const result = await session(['--for', 'claude'], async () => {});
-    expect(result.exitCode).toBe(2);
-  });
-
-  it('says the recording is being served, not recorded, so nothing looks like a fresh run', async () => {
-    const recorded = await session(['--for', 'claude'], async (proxyUrl) => {
-      await fetch(`${proxyUrl}/v1/messages`, {
-        method: 'POST',
-        headers: { 'content-type': 'application/json' },
-        body: JSON.stringify({
-          model: 'claude-opus-5',
-          messages: [{ role: 'user', content: 'x' }],
-        }),
-      });
-    });
-    lines.length = 0;
-    await session(['--replay', recorded.runId], async () => {});
-    expect(lines.join('')).toContain('egress=blocked');
-  });
-
-  it('says so plainly when the session ended having captured nothing', async () => {
-    const result = await session([], async () => {});
-    expect(result.modelExchanges).toBe(0);
-    expect(lines.join('')).toContain('capture.empty');
-  });
-});

package/packages/cli/test/attach.test.ts

@@ -1,198 +0,0 @@
-import { describe, expect, it } from 'vitest';
-import { advertisedUrl, attachExports, attachInstructions } from '../src/attach.js';
-
-/**
- * Recording an agent agenticreplay does not launch.
- *
- * Every capture mechanism here so far ends in a child process: the adapter builds an environment
- * and agenticreplay spawns the agent inside it. That is the whole design, and it has one boundary it cannot
- * cross — an agent that is not on this machine. A bot on a VPS, an agent inside a dev container or
- * a cloud sandbox, a harness someone else's CI runs: agenticreplay has no process to spawn, so it has no
- * environment to build.
- *
- * What it can still do is be *reachable*, and say exactly what to set. The proxy already accepts a
- * bind address; what was missing was the other half — the block of variables the operator pastes
- * on the far side, which is the part that has to be right or the run records nothing.
- */
-describe('the address a remote agent is told to use', () => {
-  it('uses the bind address when it is one an agent could dial', () => {
-    expect(advertisedUrl({ bind: '10.0.0.5', port: 8080 })).toBe('http://10.0.0.5:8080');
-  });
-
-  /**
-   * `0.0.0.0` means "every interface I have", which is a statement about listening. As an address
-   * to connect *to* it is meaningless, and an agent handed `http://0.0.0.0:8080` fails in a way
-   * that looks like agenticreplay is broken rather than like the flag was incomplete.
-   */
-  it('refuses to advertise a wildcard, which is not an address anything can reach', () => {
-    expect(() => advertisedUrl({ bind: '0.0.0.0', port: 8080 })).toThrow(/--advertise/);
-    expect(() => advertisedUrl({ bind: '::', port: 8080 })).toThrow(/--advertise/);
-  });
-
-  it('takes an explicit advertised host over the bind address', () => {
-    expect(
-      advertisedUrl({ bind: '0.0.0.0', port: 8080, advertise: 'agenticreplay.internal' }),
-    ).toBe('http://agenticreplay.internal:8080');
-  });
-
-  it('keeps a port the operator wrote into the advertised host', () => {
-    // Behind a port-mapped container the outside port is not the one agenticreplay bound.
-    expect(advertisedUrl({ bind: '0.0.0.0', port: 8080, advertise: 'localhost:19000' })).toBe(
-      'http://localhost:19000',
-    );
-  });
-
-  it('brackets a bare IPv6 literal, which is not a URL host without them', () => {
-    expect(advertisedUrl({ bind: 'fd00::1', port: 8080 })).toBe('http://[fd00::1]:8080');
-  });
-
-  /**
-   * An IPv6 literal is mostly colons, so "does this host already carry a port" cannot be answered
-   * by looking for the last one. Getting it wrong drops the port entirely and hands the sandbox
-   * `http://fd00::1`, which resolves to port 80 and connects to nothing.
-   */
-  it('does not mistake the tail of a bare IPv6 address for a port', () => {
-    expect(advertisedUrl({ bind: '0.0.0.0', port: 8080, advertise: 'fd00::1' })).toBe(
-      'http://[fd00::1]:8080',
-    );
-    expect(advertisedUrl({ bind: '0.0.0.0', port: 8080, advertise: '::1' })).toBe(
-      'http://[::1]:8080',
-    );
-  });
-
-  it('keeps a port written after a bracketed IPv6 host', () => {
-    expect(advertisedUrl({ bind: '0.0.0.0', port: 8080, advertise: '[fd00::1]:19000' })).toBe(
-      'http://[fd00::1]:19000',
-    );
-  });
-});
-
-describe('what a remote agent has to be told', () => {
-  const proxyUrl = 'http://10.0.0.5:8080';
-
-  it('points the agent at the proxy through the ordinary variables', () => {
-    const env = attachExports({ proxyUrl, adapterEnv: { OPENAI_BASE_URL: `${proxyUrl}/v1` } });
-    expect(env.OPENAI_BASE_URL).toBe('http://10.0.0.5:8080/v1');
-  });
-
-  it('adds the proxy and the CA when the run is intercepting', () => {
-    const env = attachExports({
-      proxyUrl,
-      adapterEnv: {},
-      ca: { certPath: '/runs/r1/tls/ca.crt', bundlePath: '/runs/r1/tls/ca-bundle.crt' },
-      remoteCaPath: '/tmp/agenticreplay-ca.crt',
-    });
-    expect(env.HTTPS_PROXY).toBe(proxyUrl);
-    expect(env.https_proxy).toBe(proxyUrl);
-    // Every path is the one the file will have *there*, never the one it has here. Printing this
-    // machine's path would produce a block that runs cleanly and trusts nothing.
-    expect(env.NODE_EXTRA_CA_CERTS).toBe('/tmp/agenticreplay-ca.crt');
-    expect(env.SSL_CERT_FILE).toBe('/tmp/agenticreplay-ca.crt');
-    expect(env.REQUESTS_CA_BUNDLE).toBe('/tmp/agenticreplay-ca.crt');
-    expect(env.CURL_CA_BUNDLE).toBe('/tmp/agenticreplay-ca.crt');
-  });
-
-  it('sets no CA variable when nothing is being intercepted', () => {
-    const env = attachExports({ proxyUrl, adapterEnv: { OPENAI_BASE_URL: `${proxyUrl}/v1` } });
-    expect(env.NODE_EXTRA_CA_CERTS).toBeUndefined();
-    expect(env.HTTPS_PROXY).toBeUndefined();
-  });
-
-  /**
-   * The block is printed to a terminal, and a terminal is scrollback, a screen share, a pasted
-   * ticket and a CI log. The operator's own credential must never appear in it.
-   *
-   * Nor should a placeholder: adapters substitute `agenticreplay-recorded` for a key the local environment
-   * does not have, and exporting that in a sandbox would *overwrite* a real key the agent already
-   * had, turning a working setup into a 401. The agent over there keeps whatever credential it
-   * has, and agenticreplay forwards it upstream — so the right number of credential lines is none.
-   */
-  it('exports no credential, neither a real one nor a placeholder', () => {
-    const env = attachExports({
-      proxyUrl,
-      adapterEnv: {
-        ANTHROPIC_BASE_URL: proxyUrl,
-        ANTHROPIC_API_KEY: 'sk-ant-REAL-SECRET',
-        OPENAI_API_KEY: 'agenticreplay-recorded',
-        GITHUB_TOKEN: 'ghp_secret',
-      },
-    });
-    expect(env.ANTHROPIC_BASE_URL).toBe(proxyUrl);
-    expect(env.ANTHROPIC_API_KEY).toBeUndefined();
-    expect(env.OPENAI_API_KEY).toBeUndefined();
-    expect(env.GITHUB_TOKEN).toBeUndefined();
-  });
-
-  it('leaves no trace of a secret in the printed block', () => {
-    const text = attachInstructions({
-      proxyUrl,
-      adapterEnv: { ANTHROPIC_BASE_URL: proxyUrl, ANTHROPIC_API_KEY: 'sk-ant-REAL-SECRET' },
-    }).join('\n');
-    expect(text).not.toContain('sk-ant-REAL-SECRET');
-    // And says why a key is missing, so nobody concludes the block is incomplete.
-    expect(text).toMatch(/credential/i);
-  });
-
-  /**
-   * A sandbox that cannot reach the proxy is the failure this whole command exists to avoid, and
-   * `NO_PROXY` inherited from the operator's shell is a good way to cause it silently.
-   */
-  it('does not carry the local NO_PROXY into the sandbox', () => {
-    const env = attachExports({
-      proxyUrl,
-      adapterEnv: {},
-      ca: { certPath: '/a/ca.crt', bundlePath: '/a/b.crt' },
-      remoteCaPath: '/tmp/ca.crt',
-    });
-    expect(env.NO_PROXY).toBeUndefined();
-  });
-});
-
-describe('the block the operator pastes', () => {
-  it('is shell-pasteable, one export per line', () => {
-    const lines = attachInstructions({
-      proxyUrl: 'http://10.0.0.5:8080',
-      adapterEnv: { OPENAI_BASE_URL: 'http://10.0.0.5:8080/v1' },
-    });
-    expect(lines).toContain("export OPENAI_BASE_URL='http://10.0.0.5:8080/v1'");
-  });
-
-  it('quotes a value so a shell cannot reinterpret it', () => {
-    const lines = attachInstructions({
-      proxyUrl: 'http://h:1',
-      adapterEnv: { WEIRD: "a b$(echo hi)'c" },
-    });
-    expect(lines.join('\n')).toContain(`'a b$(echo hi)'\\''c'`);
-  });
-
-  /**
-   * The bundle rather than the bare certificate, and the distinction is not cosmetic.
-   *
-   * `SSL_CERT_FILE` and its siblings *replace* an OpenSSL client's trust store rather than adding
-   * to it. A sandbox handed only `ca.crt` would trust the run CA and nothing else, so every host
-   * agenticreplay deliberately does not intercept — the package index, the git remote — would stop
-   * verifying. The bundle is the run CA plus the public roots, which is why one file is safe to
-   * ship and safe to point all of them at.
-   */
-  it('says to copy the CA bundle across first, because the paths are remote', () => {
-    const lines = attachInstructions({
-      proxyUrl: 'http://10.0.0.5:8080',
-      adapterEnv: {},
-      ca: { certPath: '/runs/r1/tls/ca.crt', bundlePath: '/runs/r1/tls/ca-bundle.crt' },
-      remoteCaPath: '/tmp/agenticreplay-ca.crt',
-    });
-    const text = lines.join('\n');
-    expect(text).toContain('/runs/r1/tls/ca-bundle.crt');
-    expect(text).not.toContain('/runs/r1/tls/ca.crt ');
-    expect(text).toContain('/tmp/agenticreplay-ca.crt');
-    expect(text).toMatch(/copy/i);
-  });
-
-  it('mentions no certificate authority when there is none to copy', () => {
-    const text = attachInstructions({
-      proxyUrl: 'http://h:1',
-      adapterEnv: { OPENAI_BASE_URL: 'http://h:1/v1' },
-    }).join('\n');
-    expect(text).not.toMatch(/ca-bundle\.crt/);
-  });
-});