agenticreplay 0.1.0 → 0.1.2

package/packages/adapters/test/exec-adapter.test.ts

@@ -1,114 +0,0 @@
-import { mkdtemp, rm } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-import type { RecordContext } from '@agenticreplay/plugin-api';
-import { checkAdapterContract, formatContractResult } from '../src/contract.js';
-import { defaultAdapters } from '../src/registry.js';
-import { execAdapter } from '../src/exec.js';
-
-/**
- * The adapter for an agent agenticreplay knows nothing about.
- *
- * Every other adapter here names a harness and the variable that harness reads. This one names
- * neither, because the case it exists for is an agent whose model loop agenticreplay cannot see into at
- * all: a Python bot with a hardcoded xAI origin, a Go binary, an editor that spawns its own agent
- * as a child. Capture for those happens at the transport — `--tls-intercept` — and the adapter's
- * whole job is to launch the command and get out of the way.
- *
- * The interesting assertions here are therefore about what it does *not* set. `generic-openai`
- * already covers "run my command and redirect the usual variables"; if this adapter also injected
- * an origin it would be that adapter with a different name, and it would be pointing agents at a
- * provider they never asked for.
- */
-describe('the exec adapter', () => {
-  let root: string;
-  let ctx: RecordContext;
-
-  beforeEach(async () => {
-    root = await mkdtemp(join(tmpdir(), 'agenticreplay-exec-'));
-    ctx = {
-      runId: 'run_test',
-      cwd: join(root, 'work'),
-      runDir: join(root, 'run'),
-      proxyUrl: 'http://127.0.0.1:44100',
-      userArgs: ['python', 'bot.py'],
-      env: {},
-    };
-  });
-  afterEach(async () => {
-    await rm(root, { recursive: true, force: true });
-  });
-
-  it('passes the adapter contract', async () => {
-    const result = await checkAdapterContract(execAdapter, {
-      ctx: { userArgs: ['python', 'bot.py'] },
-    });
-    expect(result.ok, formatContractResult(result)).toBe(true);
-  });
-
-  it('answers to the names people type', () => {
-    const registry = defaultAdapters();
-    expect(registry.get('exec').id).toBe('exec');
-    expect(registry.get('any').id).toBe('exec');
-  });
-
-  it('never claims a workspace, because it cannot know what it would launch', async () => {
-    expect(await execAdapter.detect(ctx.cwd)).toBe(false);
-  });
-
-  it('launches the command from argv and forwards the rest', async () => {
-    const launch = await execAdapter.prepare({
-      ...ctx,
-      userArgs: ['python', 'bot.py', '--task', 'fix it'],
-    });
-    expect(launch.command).toBe('python');
-    expect(launch.args).toEqual(['bot.py', '--task', 'fix it']);
-  });
-
-  /**
-   * The whole point of the adapter. An injected origin is not free: a harness that reads
-   * `OPENAI_BASE_URL` and would otherwise have called xAI gets silently repointed, and the run
-   * records traffic for a provider the user never chose.
-   */
-  it('injects no base-url variable of its own', async () => {
-    const launch = await execAdapter.prepare(ctx);
-    for (const name of ['OPENAI_BASE_URL', 'OPENAI_API_BASE', 'ANTHROPIC_BASE_URL']) {
-      expect(launch.env[name], `${name} must not be invented`).toBeUndefined();
-    }
-    expect(Object.keys(launch.env).filter((n) => /_BASE_URL$|_API_BASE$/.test(n))).toEqual([]);
-  });
-
-  it('invents no credential', async () => {
-    const launch = await execAdapter.prepare(ctx);
-    expect(Object.keys(launch.env).filter((n) => /KEY$|TOKEN$|SECRET$/.test(n))).toEqual([]);
-  });
-
-  /**
-   * The one redirect it will do, because the operator asked for it by name. This is what makes
-   * `exec` useful without `--tls-intercept` for a harness that reads an unusual variable.
-   */
-  it('redirects a variable the operator names', async () => {
-    const launch = await execAdapter.prepare({
-      ...ctx,
-      env: { AGENTICREPLAY_BASE_URL_VARS: 'GROK_BASE_URL' },
-    });
-    expect(launch.env.GROK_BASE_URL).toBe('http://127.0.0.1:44100/v1');
-  });
-
-  it('says what to type when the command is missing', async () => {
-    await expect(execAdapter.prepare({ ...ctx, userArgs: [] })).rejects.toThrow(
-      /agenticreplay record exec .*-- <command>/,
-    );
-  });
-
-  /**
-   * Declared rather than inferred. The contract's `redirects-model-traffic` check exists to catch
-   * an adapter that points the harness nowhere — which is precisely, and deliberately, what this
-   * adapter does. Marking it as transport-captured is how it opts out honestly, instead of setting
-   * an inert `AGENTICREPLAY_PROXY_URL` to look like it installed a hook it never installed.
-   */
-  it('declares that it captures at the transport, not through the environment', () => {
-    expect(execAdapter.capture).toBe('transport');
-  });
-});

package/packages/adapters/test/harness-versions.test.ts

@@ -1,220 +0,0 @@
-import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import type { Adapter, Launch, RecordContext } from '@agenticreplay/plugin-api';
-import { defaultAdapters } from '../src/index.js';
-
-/**
- * Recorded wire contracts, one per adapter.
- *
- * A fixture is a claim: *against these harness versions, on this date, this adapter set exactly
- * these variables to exactly these values.* The contract in contract.ts checks that an adapter is
- * still shaped correctly; these fixtures check that it has not quietly changed shape.
- *
- * **When a harness changes and someone updates an adapter, the fixture must be updated in the
- * same PR.** That is the whole point. The diff is where a reviewer sees "this adapter now sets a
- * different variable" — and `verified_at` is the reviewer's cue to ask the only question that
- * matters: did you actually run the harness, or did you guess? A fixture nobody has to touch is a
- * fixture that records what the adapter used to do.
- *
- * `verified_at` is the day someone last confirmed the adapter against a running harness.
- */
-
-const PROXY = 'http://127.0.0.1:51733';
-/** First arg is the command for adapters that take one (generic-openai); the rest is padding. */
-const USER_ARGS = ['my-agent', '--task', 'demo'];
-const FIXTURES = new URL('../fixtures/harness/', import.meta.url);
-// `_HOST` too: goose reads `OPENAI_HOST` and `ANTHROPIC_HOST` rather than the `_BASE_URL`
-// spellings, and those are the variables its whole capture depends on. Left out, the two that
-// matter most for that adapter were the two nothing checked. `AGENTICREPLAY_INSTRUMENT_HOSTS` is a list
-// of hostnames rather than an origin, and ends `HOSTS`, so it stays out.
-const BASE_URL_LIKE = /(?:_BASE_URL|_API_BASE|_HOST)$/;
-
-interface HarnessFixture {
-  adapter: string;
-  harness_versions: string | null;
-  verified_at: string;
-  command: string;
-  /** Every variable a bare run sets, sorted. The exact wire contract. */
-  env_vars: string[];
-  /** Base-url variable to the path appended to the proxy url. */
-  base_urls: Record<string, string>;
-  /** Set only when the incoming environment already carries them — never invented. */
-  optional_env_vars: string[];
-  /**
-   * Variables the adapter rewrites rather than passes on, and the proxy path each becomes.
-   *
-   * `optional_env_vars` cannot describe these: it asserts the value comes through untouched, and
-   * the whole point of a second model origin is that the value is *replaced* — with a
-   * `/forward/` path carrying the destination agenticreplay took the call away from. Without an entry
-   * here, an adapter that stopped redirecting its embedding endpoint would pass every check in
-   * this file while sending that traffic straight past the proxy.
-   */
-  redirected_env_vars?: Record<string, string>;
-  note?: string;
-}
-
-/** The origin a `redirected_env_vars` entry is probed with. Its encoding is what the path holds. */
-const REDIRECT_PROBE = 'https://embeddings.example/v1';
-
-/**
- * A real directory, because `ctx.runDir` is where the contract tells an adapter to put scratch
- * files — the MCP config rewrite and the fetch instrument both write one. With `/work` hardcoded
- * here, the first adapter to do that created a directory at the filesystem root on whoever ran
- * the tests, or failed outright on a machine where they could not.
- */
-const SCRATCH = mkdtempSync(join(tmpdir(), 'agenticreplay-harness-fixture-'));
-afterAll(() => rmSync(SCRATCH, { recursive: true, force: true }));
-
-/**
- * An empty home, saved and restored. An adapter's `prepare` may consult the real home — OpenCode's
- * does, because `opencode auth login` writes a credential file there and that file decides the
- * credential branch — so a fixture recorded on a machine that has signed in reads a different
- * contract from one that has not. The fixtures describe the adapter, not the machine running the
- * tests.
- */
-const SAVED_HOME = { HOME: process.env['HOME'], USERPROFILE: process.env['USERPROFILE'] };
-beforeAll(() => {
-  process.env['HOME'] = SCRATCH;
-  process.env['USERPROFILE'] = SCRATCH;
-});
-afterAll(() => {
-  for (const [name, value] of Object.entries(SAVED_HOME)) {
-    if (value === undefined) delete process.env[name];
-    else process.env[name] = value;
-  }
-});
-
-function ctx(over: Partial<RecordContext> = {}): RecordContext {
-  return {
-    runId: 'run_fixture',
-    cwd: join(SCRATCH, 'work'),
-    proxyUrl: PROXY,
-    runDir: join(SCRATCH, 'work', '.agenticreplay', 'runs', 'run_fixture'),
-    userArgs: [...USER_ARGS],
-    // PATH, because a real run carries one and an adapter may resolve against it — OpenCode's
-    // now points SHELL at a shim, and only when the real shell is findable again.
-    env: { PATH: process.env['PATH'] ?? '' },
-    ...over,
-  };
-}
-
-function fixturePath(id: string): string {
-  return fileURLToPath(new URL(`${id}.json`, FIXTURES));
-}
-
-/** What the adapter does today, in fixture shape — the failure message pastes straight into JSON. */
-function observed(adapter: Adapter, launch: Launch): Omit<HarnessFixture, 'verified_at'> {
-  const baseUrls: Record<string, string> = {};
-  for (const [name, value] of Object.entries(launch.env)) {
-    if (BASE_URL_LIKE.test(name)) baseUrls[name] = value.slice(PROXY.length);
-  }
-  return {
-    adapter: adapter.id,
-    harness_versions: adapter.harnessVersions ?? null,
-    command: launch.command,
-    env_vars: Object.keys(launch.env).sort(),
-    base_urls: baseUrls,
-    optional_env_vars: [],
-  };
-}
-
-function readFixture(adapter: Adapter, launch: Launch): HarnessFixture {
-  const path = fixturePath(adapter.id);
-  if (!existsSync(path)) {
-    const suggestion = { ...observed(adapter, launch), verified_at: 'YYYY-MM-DD' };
-    throw new Error(
-      `no harness fixture for '${adapter.id}'. Without one, nothing notices when the harness ` +
-        `changes what it reads. Create fixtures/harness/${adapter.id}.json:\n` +
-        `${JSON.stringify(suggestion, null, 2)}\n` +
-        'setting verified_at to the day you ran the harness and harness_versions to the range ' +
-        'you ran it against.',
-    );
-  }
-  return JSON.parse(readFileSync(path, 'utf8')) as HarnessFixture;
-}
-
-const registry = defaultAdapters();
-
-describe.each(registry.ids())('%s', (id) => {
-  const adapter = registry.get(id);
-  // This historical fixture describes OpenCode 1.18.29, independently of the installed CLI.
-  // Version 2 has separate native and launch regressions.
-  function fixtureContext(over: Partial<RecordContext> = {}): RecordContext {
-    const context = ctx(over);
-    if (id === 'opencode') context.env.AGENTICREPLAY_OPENCODE_VERSION = '1.18.29';
-    return context;
-  }
-
-  /**
-   * The fixtures were captured on POSIX, and one variable in them is the host's, not agenticreplay's:
-   * `opencode` passes `$SHELL` through, and Windows does not set it. Comparing the whole key list
-   * on Windows therefore fails over the absence of a variable agenticreplay never promised to invent.
-   *
-   * Skipped rather than filtered so the fixture keeps meaning what it says — it is a record of what
-   * this adapter produced on the platform it was captured on, and CI runs that platform.
-   */
-  it.skipIf(process.platform === 'win32')(
-    'still produces the env it was recorded with',
-    async () => {
-      const launch = await adapter.prepare(fixtureContext());
-      const fixture = readFixture(adapter, launch);
-      expect(Object.keys(launch.env).sort()).toEqual(fixture.env_vars);
-      expect(launch.command).toBe(fixture.command);
-    },
-  );
-
-  it('still points every base url at the proxy, on the recorded path', async () => {
-    const launch = await adapter.prepare(fixtureContext());
-    const fixture = readFixture(adapter, launch);
-    for (const [name, path] of Object.entries(fixture.base_urls)) {
-      expect(launch.env[name]).toBe(`${PROXY}${path}`);
-    }
-    // A base-url variable the adapter grew without a fixture entry is exactly the silent change
-    // this file exists to catch.
-    const live = Object.keys(launch.env).filter((name) => BASE_URL_LIKE.test(name));
-    expect(live.sort()).toEqual(Object.keys(fixture.base_urls).sort());
-  });
-
-  it('still redirects the origins it is recorded as redirecting', async () => {
-    const fixture = readFixture(adapter, await adapter.prepare(fixtureContext()));
-    for (const [name, path] of Object.entries(fixture.redirected_env_vars ?? {})) {
-      const launch = await adapter.prepare(fixtureContext({ env: { [name]: REDIRECT_PROBE } }));
-      expect(launch.env[name]).toBe(`${PROXY}${path}`);
-    }
-  });
-
-  it('passes its optional variables through without inventing them', async () => {
-    const bare = await adapter.prepare(fixtureContext());
-    const fixture = readFixture(adapter, bare);
-    for (const name of fixture.optional_env_vars) {
-      expect(Object.keys(bare.env)).not.toContain(name);
-      const carried = await adapter.prepare(fixtureContext({ env: { [name]: 'from-the-user' } }));
-      expect(carried.env[name]).toBe('from-the-user');
-    }
-  });
-
-  it('records the harness versions the adapter declares, so the two cannot drift', async () => {
-    const fixture = readFixture(adapter, await adapter.prepare(fixtureContext()));
-    expect(fixture.harness_versions).toBe(adapter.harnessVersions ?? null);
-    expect(fixture.adapter).toBe(id);
-  });
-
-  it('carries a real verification date', async () => {
-    const fixture = readFixture(adapter, await adapter.prepare(fixtureContext()));
-    expect(fixture.verified_at).toMatch(/^\d{4}-\d{2}-\d{2}$/);
-    expect(Number.isNaN(Date.parse(fixture.verified_at))).toBe(false);
-  });
-});
-
-describe('the fixture directory', () => {
-  it('has one fixture per registered adapter and no orphans', () => {
-    const files = readdirSync(fileURLToPath(FIXTURES))
-      .filter((f) => f.endsWith('.json'))
-      .map((f) => f.replace(/\.json$/, ''))
-      .sort();
-    expect(files).toEqual([...registry.ids()].sort());
-  });
-});

package/packages/adapters/test/indexrag.test.ts

@@ -1,192 +0,0 @@
-import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-import type { RecordContext } from '@agenticreplay/plugin-api';
-import { decodeForwardPath } from '@agenticreplay/proxy';
-import { checkAdapterContract, formatContractResult } from '../src/contract.js';
-import { indexRagAdapter } from '../src/indexrag.js';
-import { defaultAdapters } from '../src/registry.js';
-
-/**
- * The first adapter for a pipeline rather than a conversational agent.
- *
- * Three things are different from every adapter before it, and each has a test here: two model
- * origins in one run, a product that lives outside the tracked working tree, and a harness that
- * resumes from that product rather than redoing the work.
- */
-describe('the indexrag adapter', () => {
-  let root: string;
-  let ctx: RecordContext;
-
-  beforeEach(async () => {
-    root = await mkdtemp(join(tmpdir(), 'agenticreplay-indexrag-'));
-    await mkdir(join(root, 'work'), { recursive: true });
-    await mkdir(join(root, 'run'), { recursive: true });
-    ctx = {
-      runId: 'run_test',
-      cwd: join(root, 'work'),
-      runDir: join(root, 'run'),
-      proxyUrl: 'http://127.0.0.1:44100',
-      userArgs: [],
-      env: {},
-    };
-  });
-  afterEach(async () => {
-    await rm(root, { recursive: true, force: true });
-  });
-
-  it('passes the adapter contract', async () => {
-    const result = await checkAdapterContract(indexRagAdapter);
-    expect(result.ok, formatContractResult(result)).toBe(true);
-  });
-
-  it('is registered under the name a user types', () => {
-    expect(defaultAdapters().get('indexrag').id).toBe('indexrag');
-  });
-
-  describe('detect', () => {
-    /** The two files the detector looks for, in a directory shaped like the source tree. */
-    async function sourceTree(dir: string): Promise<void> {
-      await mkdir(join(dir, 'indexrag', 'aku'), { recursive: true });
-      await mkdir(join(dir, 'scripts'), { recursive: true });
-      await writeFile(join(dir, 'indexrag', 'aku', 'extractor.py'), '', 'utf8');
-      await writeFile(join(dir, 'scripts', 'build_kb.py'), '', 'utf8');
-    }
-
-    it('claims the source tree', async () => {
-      await sourceTree(ctx.cwd);
-      expect(await indexRagAdapter.detect(ctx.cwd)).toBe(true);
-    });
-
-    it('declines an empty directory', async () => {
-      expect(await indexRagAdapter.detect(ctx.cwd)).toBe(false);
-    });
-
-    it('declines a directory that has only half of it', async () => {
-      // A directory called `indexrag` is a common enough name to be someone else's; both files
-      // have to be there before this adapter claims a workspace it would then launch python in.
-      await mkdir(join(ctx.cwd, 'indexrag', 'aku'), { recursive: true });
-      await writeFile(join(ctx.cwd, 'indexrag', 'aku', 'extractor.py'), '', 'utf8');
-      expect(await indexRagAdapter.detect(ctx.cwd)).toBe(false);
-    });
-
-    it('declines a path that does not exist rather than throwing', async () => {
-      expect(await indexRagAdapter.detect(join(root, 'nowhere'))).toBe(false);
-    });
-  });
-
-  describe('the two model origins', () => {
-    it('points the chat half at the proxy under /v1', async () => {
-      const launch = await indexRagAdapter.prepare(ctx);
-      expect(launch.env.OPENAI_BASE_URL).toBe('http://127.0.0.1:44100/v1');
-      // langchain's own embedding client reads the other spelling, so both are set.
-      expect(launch.env.OPENAI_API_BASE).toBe('http://127.0.0.1:44100/v1');
-    });
-
-    /**
-     * The second origin keeps its destination. AgenticReplay's upstream map is keyed by wire dialect, so
-     * "chat at a gateway, embeddings at a dedicated provider" cannot be expressed by any
-     * `--upstream-*`; the request has to carry its own address or the proxy has to guess, and
-     * guessing sent the embedding provider's key to `api.openai.com`.
-     */
-    it('redirects the embedding origin through a forward path that names it', async () => {
-      const launch = await indexRagAdapter.prepare({
-        ...ctx,
-        env: { INDEXRAG_EMBEDDING_BASE_URL: 'https://maas.example/v1' },
-      });
-      const value = launch.env.INDEXRAG_EMBEDDING_BASE_URL!;
-      expect(value.startsWith('http://127.0.0.1:44100/forward/')).toBe(true);
-      expect(decodeForwardPath(new URL(value).pathname)?.base).toBe('https://maas.example/v1');
-    });
-
-    it('leaves the embedding origin alone when the user set none', async () => {
-      const launch = await indexRagAdapter.prepare(ctx);
-      expect(launch.env.INDEXRAG_EMBEDDING_BASE_URL).toBeUndefined();
-    });
-
-    it('carries the embedding credential without inventing one', async () => {
-      expect((await indexRagAdapter.prepare(ctx)).env.INDEXRAG_EMBEDDING_API_KEY).toBeUndefined();
-      const carried = await indexRagAdapter.prepare({
-        ...ctx,
-        env: { INDEXRAG_EMBEDDING_API_KEY: 'ak-real' },
-      });
-      expect(carried.env.INDEXRAG_EMBEDDING_API_KEY).toBe('ak-real');
-    });
-
-    it('sets no Anthropic variables — every stage speaks OpenAI', async () => {
-      const launch = await indexRagAdapter.prepare(ctx);
-      expect(Object.keys(launch.env).filter((k) => k.startsWith('ANTHROPIC_'))).toEqual([]);
-    });
-  });
-
-  describe('what it launches', () => {
-    it('drives the whole pipeline when the user passed only flags', async () => {
-      const launch = await indexRagAdapter.prepare({
-        ...ctx,
-        userArgs: ['--data-dir', 'dataset/mini/documents'],
-      });
-      expect(launch.command).toBe('python');
-      expect(launch.args[0]).toBe(join(ctx.runDir, 'drive_indexrag.py'));
-      expect(launch.args.slice(1)).toEqual(['--data-dir', 'dataset/mini/documents']);
-      // Written where the contract allows scratch, and declared so the run can clean it up.
-      expect(launch.tempFiles).toEqual([launch.args[0]]);
-      const driver = await readFile(launch.args[0]!, 'utf8');
-      expect(driver).toContain('scripts.extract_akus');
-      expect(driver).toContain('scripts.generate_bridging');
-      expect(driver).toContain('scripts.build_kb');
-      expect(driver).toContain('benchmarks.evaluate');
-    });
-
-    it('runs one stage when the user named a command', async () => {
-      // What someone debugging the query half wants: record the answering stage on its own.
-      const launch = await indexRagAdapter.prepare({
-        ...ctx,
-        userArgs: ['python', '-m', 'benchmarks.evaluate', '--dataset', 'mini'],
-      });
-      expect(launch.command).toBe('python');
-      expect(launch.args).toEqual(['-m', 'benchmarks.evaluate', '--dataset', 'mini']);
-      expect(launch.tempFiles).toBeUndefined();
-    });
-
-    it('drives the pipeline when the user passed nothing at all', async () => {
-      const launch = await indexRagAdapter.prepare(ctx);
-      expect(launch.command).toBe('python');
-      expect(launch.args).toEqual([join(ctx.runDir, 'drive_indexrag.py')]);
-    });
-  });
-
-  describe('its artifacts', () => {
-    it('captures the index, the caches, and the corpus its .gitignore excludes', () => {
-      expect(indexRagAdapter.artifacts?.capture).toContain('cache');
-      expect(indexRagAdapter.artifacts?.capture).toContain('vector_store');
-      // The input, not just the output. Without it a replay anywhere but the recording's own
-      // directory starts with no documents and reproduces nothing.
-      expect(indexRagAdapter.artifacts?.capture).toContain('dataset');
-    });
-
-    it('never resets the corpus, which is input rather than product', () => {
-      expect(indexRagAdapter.artifacts?.resetBeforeReplay).not.toContain('dataset');
-    });
-
-    /**
-     * Everything reset must also be captured, or a replay deletes something the safety snapshot
-     * never took a copy of — which is the operator's index gone for good.
-     */
-    it('resets nothing it does not also capture', () => {
-      const captured = new Set(indexRagAdapter.artifacts?.capture ?? []);
-      for (const path of indexRagAdapter.artifacts?.resetBeforeReplay ?? []) {
-        expect(captured.has(path), `${path} is reset but never captured`).toBe(true);
-      }
-    });
-
-    it('names the flag that would serialise the pipeline, without applying it', () => {
-      expect(indexRagAdapter.artifacts?.concurrencyFlag).toEqual({
-        flag: '--concurrency',
-        serialValue: '1',
-      });
-      // Declared, never imposed: clamping concurrency would change the run being replayed.
-      expect(indexRagAdapter.replayArgs).toBeUndefined();
-    });
-  });
-});