agenticreplay 0.1.0 → 0.1.2

package/packages/cli/test/flags.test.ts

@@ -1,313 +0,0 @@
-import { existsSync, readFileSync, readdirSync } from 'node:fs';
-import { dirname, join } from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { describe, expect, it } from 'vitest';
-import { VALUELESS, parseArgs } from '../src/args.js';
-import {
-  BY_COMMAND,
-  GLOBAL,
-  POSITIONAL_COUNTS,
-  assertKnownFlags,
-  assertNoStrayPositionals,
-} from '../src/flags.js';
-
-/**
- * A flag is an instruction. The parser takes any it is given, which is right for parsing and wrong
- * for a CLI: an invented one was accepted in silence and simply had no effect, so a run that did
- * something other than what was asked reported success. Someone told to use a flag that does not
- * exist got a normal run and no hint of it.
- */
-describe('assertKnownFlags', () => {
-  const check = (argv: string[]) => () => assertKnownFlags(parseArgs(argv));
-
-  it('accepts the flags a command actually has', () => {
-    expect(check(['replay', 'last', '--from', '4', '--model', 'x', '--in-place'])).not.toThrow();
-    expect(check(['record', 'claude', '--no-shell', '--mcp-config', 'm.json'])).not.toThrow();
-    expect(check(['gc', '--older-than', '7d', '--dry-run'])).not.toThrow();
-  });
-
-  it('rejects one the command does not have, and says so', () => {
-    expect(check(['replay', 'last', '--explain-miss'])).toThrow(/unknown flag --explain-miss/);
-  });
-
-  // A flag that exists somewhere else is exactly the kind that reads as working.
-  it('rejects a flag that belongs to another command', () => {
-    expect(check(['list', '--from', '4'])).toThrow(/unknown flag --from/);
-  });
-
-  it('names the nearest flag when it looks like a typo', () => {
-    expect(check(['replay', 'last', '--worktre'])).toThrow(/did you mean --worktree/);
-  });
-
-  it('lists what the command does take, so the error is enough to act on', () => {
-    expect(check(['scrub', 'last', '--nope'])).toThrow(/--match --matches --dry-run --drop-fs/);
-  });
-
-  it('allows the global flags everywhere', () => {
-    expect(check(['show', 'last', '--json', '--verbose', '--ci'])).not.toThrow();
-  });
-
-  // `--no-shell` is `shell: false`; the negation must not read as a flag named `no-shell`.
-  it('understands a negated flag as the flag it negates', () => {
-    expect(check(['record', 'claude', '--no-fs', '--no-shell'])).not.toThrow();
-    expect(check(['replay', 'last', '--no-trace'])).not.toThrow();
-  });
-
-  /**
-   * The list is built from what the code reads, not from what the docs describe. `--no-trace` is
-   * a replay flag and record never looks at it, so record has to reject it — the alternative is
-   * the shape of failure this check exists for, with the flag ignored and the run reporting fine.
-   */
-  it('rejects a real flag on a command that does not read it', () => {
-    expect(check(['record', 'claude', '--no-trace'])).toThrow(/unknown flag --trace/);
-  });
-
-  it('says nothing about an unknown command, which the dispatcher reports better', () => {
-    expect(check(['bogus', '--whatever'])).not.toThrow();
-  });
-});
-
-/**
- * The list is hand-written, and a hand-written mirror of the code drifts the moment the code moves.
- * These two read the source and hold it to the same standard the file claims for itself.
- *
- * Both were written after the list shipped with real holes in it: `agenticreplay export --card` — a picture
- * command the README documents and `inspect.ts` implements — was rejected outright, and `graph`
- * had no entry at all, so every flag it took went unchecked. One file, `inspect.ts`, serves show,
- * graph, export and ui, and the list was built a command at a time; the flags of the other three
- * were simply never looked for.
- */
-describe('the allowlist against the source it mirrors', () => {
-  const SRC = join(dirname(fileURLToPath(import.meta.url)), '..', 'src');
-
-  /** Every `args.bool|str|num|list('name')` in the CLI's own source. */
-  function flagsReadInSource(): Set<string> {
-    const found = new Set<string>();
-    const walk = (dir: string): void => {
-      for (const entry of readdirSync(dir, { withFileTypes: true })) {
-        const path = join(dir, entry.name);
-        if (entry.isDirectory()) walk(path);
-        else if (entry.name.endsWith('.ts')) {
-          const src = readFileSync(path, 'utf8');
-          for (const m of src.matchAll(/args\.(?:bool|str|num|list)\('([^']+)'\)/g)) {
-            found.add(m[1]!);
-          }
-        }
-      }
-    };
-    walk(SRC);
-    return found;
-  }
-
-  it('allows every flag the CLI actually reads somewhere', () => {
-    const allowed = new Set<string>([...GLOBAL, ...Object.values(BY_COMMAND).flat()]);
-    const unreachable = [...flagsReadInSource()].filter((name) => !allowed.has(name)).sort();
-    // A flag the code reads and no command allows can never be passed: the command throws first.
-    expect(unreachable).toEqual([]);
-  });
-
-  /**
-   * The one above flattens every command's flags into one set and asks whether SOMEONE allows the
-   * name. That is too weak by exactly one level, and the gap shipped: `push` reads `args.bool('fs')`
-   * and the help text offers `--fs`, but `push`'s own entry listed only `gateway` and `force`.
-   * Because `record` and `replay` both allow `fs`, the union check was satisfied and said nothing.
-   *
-   * What the user got was not a flag quietly ignored — it was `agenticreplay push --fs` REFUSED, with an
-   * error naming only the two flags that were listed. The snapshots could therefore not be shipped
-   * by any invocation, so every run on a gateway carried `blob_count: 0`, and forking a pulled run
-   * died inside `git read-tree` on a tree object that had never left the machine.
-   *
-   * The help text is the contract the user reads, so hold the allowlist to it per command rather
-   * than in aggregate. `sync.test.ts` could not catch this: it calls `pushCommand` with args it
-   * built itself, which never passes through the gate this file guards.
-   */
-  it('accepts, for each command, every flag the help text offers it', () => {
-    const help = readFileSync(join(SRC, 'main.ts'), 'utf8').split(/\r?\n/);
-    const documented = new Map<string, Set<string>>();
-    let command: string | undefined;
-    for (const line of help) {
-      // `  agenticreplay push [run]   send a run to the gateway` — two spaces, then the command.
-      const heading = line.match(/^ {2}agenticreplay ([a-z-]+)\b(.*)$/);
-      if (heading) {
-        command = heading[1]!;
-        const flags = documented.get(command) ?? new Set<string>();
-        // A usage line can carry a flag itself: `agenticreplay replay [run] --from N`.
-        for (const m of heading[2]!.matchAll(/--([a-z][a-z0-9-]*)/g)) flags.add(m[1]!);
-        documented.set(command, flags);
-        continue;
-      }
-      // `        --fs   include the workspace snapshots` — eight spaces, under the last heading.
-      const flag = command === undefined ? null : line.match(/^ {8}--([a-z][a-z0-9-]*)/);
-      // `--no-fs` is `fs` as far as the allowlist is concerned: the parser strips the prefix and
-      // records the base name (see `args.ts`), so checking the spelling the help shows would fail
-      // on every negated flag the help documents.
-      if (flag) documented.get(command!)!.add(flag[1]!.replace(/^no-/, ''));
-    }
-    // The parse itself has to keep working, or the check passes by finding nothing.
-    expect(documented.get('push'), 'the help block stopped parsing').toBeDefined();
-    expect([...documented.get('push')!].sort()).toContain('fs');
-
-    const refused: string[] = [];
-    for (const [cmd, flags] of documented) {
-      const allowed = new Set<string>([...(BY_COMMAND[cmd] ?? []), ...GLOBAL]);
-      for (const name of flags)
-        if (!allowed.has(name)) refused.push(`agenticreplay ${cmd} --${name}`);
-    }
-    expect(refused.sort()).toEqual([]);
-  });
-
-  /**
-   * A flag the code reads as a boolean has to be declared valueless, or the parser hands it the
-   * next word and the positional it belonged to disappears.
-   *
-   * That is the failure `VALUELESS` was written for, and it happened again the moment a flag was
-   * added without an entry: `agenticreplay replay --quiet run_a1b2c3` parsed as `quiet="run_a1b2c3"` with an
-   * empty positional list, so replay fell back to `last` and reproduced a different run than the
-   * one named — silently, and `--quiet` had by then suppressed the output that might have given it
-   * away. Reading the source rather than restating the list is the point: the list can only stay
-   * right if adding a flag without registering it fails here.
-   */
-  it('registers every flag the code reads as a boolean, or the parser eats the next word', () => {
-    const asBoolean = new Set<string>();
-    const walk = (dir: string): void => {
-      for (const entry of readdirSync(dir, { withFileTypes: true })) {
-        const path = join(dir, entry.name);
-        if (entry.isDirectory()) walk(path);
-        else if (entry.name.endsWith('.ts')) {
-          for (const m of readFileSync(path, 'utf8').matchAll(/args\.bool\('([^']+)'/g)) {
-            asBoolean.add(m[1]!);
-          }
-        }
-      }
-    };
-    walk(SRC);
-    expect([...asBoolean].filter((name) => !VALUELESS.has(name)).sort()).toEqual([]);
-  });
-
-  it('has an entry for every command the dispatcher handles', () => {
-    const main = readFileSync(join(SRC, 'main.ts'), 'utf8');
-    const dispatched = [...main.matchAll(/case '([a-z-]+)':/g)].map((m) => m[1]!);
-    // Without an entry, assertKnownFlags returns early and the command validates nothing at all —
-    // silently, which is the failure this whole file exists to prevent.
-    const unvalidated = dispatched.filter((c) => BY_COMMAND[c] === undefined).sort();
-    expect(unvalidated).toEqual([]);
-  });
-});
-
-/**
- * The other half of the command line. `agenticreplay record codex exec "fix auth.ts"` recorded
- * `argv: ["codex"]` — the agent started with no arguments at all, did nothing it was asked, and
- * the run was reported as a success. Nothing in the output said the words had been dropped.
- */
-describe('assertNoStrayPositionals', () => {
-  const check = (argv: string[]) => () => assertNoStrayPositionals(parseArgs(argv));
-
-  it('rejects the agent arguments that were silently dropped, and shows where they go', () => {
-    expect(check(['record', 'codex', 'exec', 'fix auth.ts'])).toThrow(
-      /unexpected arguments for "agenticreplay record": exec "fix auth\.ts"/,
-    );
-    expect(check(['record', 'codex', 'exec', 'fix auth.ts'])).toThrow(
-      /agenticreplay record codex -- exec "fix auth\.ts"/,
-    );
-  });
-
-  it('leaves the correct form alone, because that is where the agent arguments belong', () => {
-    expect(check(['record', 'codex', '--', 'exec', 'fix auth.ts'])).not.toThrow();
-    expect(check(['record', 'claude'])).not.toThrow();
-    expect(check(['record'])).not.toThrow();
-  });
-
-  it('rejects a stray on a read command too, without offering the -- advice', () => {
-    expect(check(['show', 'last', 'extra'])).toThrow(
-      /unexpected argument for "agenticreplay show": extra/,
-    );
-    expect(check(['show', 'last', 'extra'])).not.toThrow(/--/);
-  });
-
-  /**
-   * Seven commands read no positional at all, so allowing one everywhere would have left the same
-   * silence in place for them. `agenticreplay attach claude` reads as naming the agent and does not: the
-   * flag that names it is `--for`.
-   */
-  it('rejects the first argument on a command that reads none', () => {
-    expect(check(['attach', 'claude'])).toThrow(
-      /unexpected argument for "agenticreplay attach": claude/,
-    );
-    expect(check(['attach', 'claude'])).toThrow(
-      /takes no arguments; everything it needs is a flag/,
-    );
-    expect(check(['list', 'foo'])).toThrow(/unexpected argument for "agenticreplay list": foo/);
-    expect(check(['doctor', 'foo'])).toThrow(/unexpected argument/);
-  });
-
-  it('leaves those commands alone when they are given none', () => {
-    expect(check(['attach', '--for', 'claude'])).not.toThrow();
-    expect(check(['list'])).not.toThrow();
-    expect(check(['gc', '--older-than', '7d'])).not.toThrow();
-  });
-
-  it('says nothing about an unknown command, which the dispatcher reports better', () => {
-    expect(check(['bogus', 'a', 'b'])).not.toThrow();
-  });
-
-  /**
-   * The rule is "one positional", and it holds only while no command reads further. Asserted
-   * against the source rather than trusted, so a command that starts taking two arguments fails
-   * here instead of having them rejected at the door.
-   */
-  it('holds because no command reads past the first positional', () => {
-    const src = join(dirname(fileURLToPath(import.meta.url)), '..', 'src');
-    const beyondFirst: string[] = [];
-    const walk = (dir: string): void => {
-      for (const entry of readdirSync(dir, { withFileTypes: true })) {
-        const path = join(dir, entry.name);
-        if (entry.isDirectory()) walk(path);
-        else if (entry.name.endsWith('.ts')) {
-          for (const m of readFileSync(path, 'utf8').matchAll(/positionals\[(\d+)\]/g)) {
-            if (Number(m[1]) > 0) beyondFirst.push(`${entry.name}: positionals[${m[1]}]`);
-          }
-        }
-      }
-    };
-    walk(src);
-    expect(beyondFirst).toEqual([]);
-  });
-});
-
-/**
- * The counts are hand-written and the code they describe is not. These hold one against the other
- * so a command that starts or stops reading a positional fails here rather than silently dropping
- * an argument again.
- */
-describe('the positional counts against the source they mirror', () => {
-  const SRC = join(dirname(fileURLToPath(import.meta.url)), '..', 'src');
-
-  it('gives a count to every command the flag list knows', () => {
-    const missing = Object.keys(BY_COMMAND)
-      .filter((c) => POSITIONAL_COUNTS[c] === undefined)
-      .sort();
-    expect(missing).toEqual([]);
-  });
-
-  it('never claims a command reads more than the source does', () => {
-    // A count above 1 would mean somewhere reads `positionals[1]`, and nothing does.
-    const overclaimed = Object.entries(POSITIONAL_COUNTS)
-      .filter(([, n]) => n > 1)
-      .map(([c]) => c);
-    expect(overclaimed).toEqual([]);
-  });
-
-  it('claims zero only for commands whose implementation reads none', () => {
-    // One file per command where the mapping is one to one; inspect.ts and main.ts serve several,
-    // so they are excluded rather than guessed at.
-    const wrong: string[] = [];
-    for (const [command, n] of Object.entries(POSITIONAL_COUNTS)) {
-      const file = join(SRC, 'commands', `${command}.ts`);
-      if (!existsSync(file)) continue;
-      const reads = readFileSync(file, 'utf8').includes('positionals[0]');
-      if (reads !== (n === 1))
-        wrong.push(`${command}: table says ${n}, source ${reads ? 'reads' : 'does not read'} one`);
-    }
-    expect(wrong).toEqual([]);
-  });
-});

package/packages/cli/test/fork-provenance.test.ts

@@ -1,224 +0,0 @@
-import { execFile } from 'node:child_process';
-import { mkdtemp, rm, writeFile } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { promisify } from 'node:util';
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-import { TraceReader, deriveCheckpoints, resolveRunSelector } from '@agenticreplay/core';
-import { validateManifest } from '@agenticreplay/schema';
-import { parseArgs } from '../src/args.js';
-import { Output } from '../src/out.js';
-import { recordCommand } from '../src/commands/record.js';
-import { replayCommand } from '../src/commands/replay.js';
-import { startFakeModel } from './fixtures/fake-model.mjs';
-
-const run = promisify(execFile);
-const FAKE_AGENT = join(import.meta.dirname, 'fixtures', 'fake-agent.mjs');
-
-/**
- * The spec gives the manifest `parent_run`, `fork_point` and `fork_model`, and they were dead:
- * fork provenance existed only as an event attribute. Anything reading a fork's manifest — `agenticreplay
- * gc` deciding what it may delete, a third-party tool, the Python SDK — saw an orphan.
- */
-describe('fork provenance in the manifest', () => {
-  let workspace: string;
-  let model: Awaited<ReturnType<typeof startFakeModel>>;
-  let out: Output;
-
-  beforeEach(async () => {
-    workspace = await mkdtemp(join(tmpdir(), 'agenticreplay-prov-'));
-    model = await startFakeModel();
-    out = new Output({ write: () => {}, isTTY: false });
-    await run('git', ['init', '-q'], { cwd: workspace });
-    await run('git', ['config', 'user.email', 't@e.com'], { cwd: workspace });
-    await run('git', ['config', 'user.name', 'T'], { cwd: workspace });
-    await writeFile(join(workspace, 'auth.ts'), 'export const fixed = false;\n');
-    process.env.FAKE_AGENT_TURNS = '3';
-    process.env.FAKE_AGENT_CWD = workspace;
-  });
-
-  afterEach(async () => {
-    await model.close();
-    await rm(workspace, { recursive: true, force: true });
-  });
-
-  async function recordThenFork(extra: string[] = []) {
-    const parent = await recordCommand(
-      parseArgs([
-        'record',
-        'generic-openai',
-        '--upstream-anthropic',
-        model.url,
-        '--',
-        'node',
-        FAKE_AGENT,
-      ]),
-      out,
-      workspace,
-    );
-    const events = await (await TraceReader.open(parent.runDir)).events();
-    const from = deriveCheckpoints(events)[0]!.seq;
-    const fork = await replayCommand(
-      parseArgs([
-        'replay',
-        parent.runId,
-        '--from',
-        String(from),
-        '--upstream-anthropic',
-        model.url,
-        ...extra,
-      ]),
-      out,
-      workspace,
-    );
-    const forkDir = (await resolveRunSelector(workspace, fork.forkRunId!)).dir;
-    return { parent, fork, from, manifest: (await TraceReader.open(forkDir)).manifest() };
-  }
-
-  it("writes the parent run id into the fork's manifest", async () => {
-    const { parent, manifest } = await recordThenFork();
-    expect(manifest.parent_run).toBe(parent.runId);
-  });
-
-  it('writes the fork point', async () => {
-    const { from, manifest } = await recordThenFork();
-    expect(manifest.fork_point).toBe(from);
-  });
-
-  it('writes the substituted model when one was given', async () => {
-    const { manifest } = await recordThenFork(['--model', 'glm-5.3-flash']);
-    expect(manifest.fork_model).toBe('glm-5.3-flash');
-  });
-
-  it('leaves all three absent on a plain recording, not null or empty', async () => {
-    const parent = await recordCommand(
-      parseArgs([
-        'record',
-        'generic-openai',
-        '--upstream-anthropic',
-        model.url,
-        '--',
-        'node',
-        FAKE_AGENT,
-      ]),
-      out,
-      workspace,
-    );
-    const manifest = (await TraceReader.open(parent.runDir)).manifest();
-    expect(manifest.parent_run).toBeUndefined();
-    expect(manifest.fork_point).toBeUndefined();
-    expect(manifest.fork_model).toBeUndefined();
-  });
-
-  it('keeps the manifest valid against the normative schema', async () => {
-    const { manifest } = await recordThenFork(['--model', 'glm-5.3-flash']);
-    const result = validateManifest(manifest);
-    expect(result.valid, result.errors.join(', ')).toBe(true);
-  });
-
-  it('reports the run it wrote under the name both replay modes share', async () => {
-    // An exact replay writes a trace of its own findings and reports it as `traceRunId`. A fork
-    // writes a run too, so the field has to mean the same thing here — otherwise a caller asking
-    // "what did this invocation produce" gets an answer only half the time. `forkRunId` stays as
-    // the narrower claim: there is a *fork*, with a worktree and exchanges of its own.
-    const { fork } = await recordThenFork();
-    expect(fork.traceRunId).toBe(fork.forkRunId);
-  });
-
-  it('still records the fork event, so both sources agree', async () => {
-    const { parent, fork, from } = await recordThenFork();
-    const forkDir = (await resolveRunSelector(workspace, fork.forkRunId!)).dir;
-    const events = await (await TraceReader.open(forkDir)).events();
-    const forkEvent = events.find((e) => e.type === 'fork')!;
-    expect(forkEvent.attrs?.parent_run).toBe(parent.runId);
-    expect(forkEvent.attrs?.fork_point).toBe(from);
-  });
-});
-
-/**
- * A fork records its own conversation, so it is a run you can read, fork again and graph — and it
- * was writing every model exchange with no `causes` at all. The recorder resolved them and the
- * fork path, a second call site of the same deriver, silently did not.
- */
-describe('a fork carries the same causal edges a recording does', () => {
-  let workspace: string;
-  let model: Awaited<ReturnType<typeof startFakeModel>>;
-  let out: Output;
-
-  beforeEach(async () => {
-    workspace = await mkdtemp(join(tmpdir(), 'agenticreplay-fork-edges-'));
-    model = await startFakeModel();
-    out = new Output({ write: () => {}, isTTY: false });
-    await run('git', ['init', '-q'], { cwd: workspace });
-    await run('git', ['config', 'user.email', 't@e.com'], { cwd: workspace });
-    await run('git', ['config', 'user.name', 'T'], { cwd: workspace });
-    await writeFile(join(workspace, 'auth.ts'), 'export const fixed = false;\n');
-    process.env.FAKE_AGENT_TURNS = '3';
-    process.env.FAKE_AGENT_CWD = workspace;
-  });
-
-  afterEach(async () => {
-    await model.close();
-    await rm(workspace, { recursive: true, force: true });
-  });
-
-  async function forkEvents() {
-    const parent = await recordCommand(
-      parseArgs([
-        'record',
-        'generic-openai',
-        '--upstream-anthropic',
-        model.url,
-        '--',
-        'node',
-        FAKE_AGENT,
-      ]),
-      out,
-      workspace,
-    );
-    const events = await (await TraceReader.open(parent.runDir)).events();
-    const from = deriveCheckpoints(events)[0]!.seq;
-    const fork = await replayCommand(
-      parseArgs([
-        'replay',
-        parent.runId,
-        '--from',
-        String(from),
-        '--upstream-anthropic',
-        model.url,
-      ]),
-      out,
-      workspace,
-    );
-    const dir = (await resolveRunSelector(workspace, fork.forkRunId!)).dir;
-    return (await TraceReader.open(dir)).events();
-  }
-
-  it('points a tool.call at the model.response that emitted it', async () => {
-    const events = await forkEvents();
-    const bySeq = new Map(events.map((e) => [e.seq, e]));
-    const calls = events.filter((e) => e.type === 'tool.call');
-    expect(calls.length).toBeGreaterThan(0);
-    for (const call of calls) {
-      expect(call.causes, `tool.call seq ${call.seq} names no cause`).toHaveLength(1);
-      expect(bySeq.get(call.causes![0]!)?.type).toBe('model.response');
-    }
-  });
-
-  it('points a tool.result back at its call, which it never did either', async () => {
-    const events = await forkEvents();
-    const bySeq = new Map(events.map((e) => [e.seq, e]));
-    const results = events.filter((e) => e.type === 'tool.result');
-    expect(results.length).toBeGreaterThan(0);
-    for (const result of results) {
-      expect(result.causes).toHaveLength(1);
-      expect(bySeq.get(result.causes![0]!)?.type).toBe('tool.call');
-    }
-  });
-
-  it('keeps every edge pointing backwards, as spec §2.1 requires', async () => {
-    for (const event of await forkEvents()) {
-      for (const cause of event.causes ?? []) expect(cause).toBeLessThan(event.seq);
-    }
-  });
-});