agenticreplay 0.1.0 → 0.1.2

package/packages/cli/src/config.ts

@@ -1,182 +0,0 @@
-import { randomBytes } from 'node:crypto';
-import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
-import { homedir } from 'node:os';
-import { dirname, join } from 'node:path';
-import { restrictToOwner } from '@agenticreplay/core';
-import type { ParsedArgs } from './args.js';
-// fetchPinned lives in @agenticreplay/core so the CLI and the recording proxy cannot
-// each carry their own copy — see its doc comment.
-export { fetchPinned } from '@agenticreplay/core';
-
-/**
- * User-level configuration, which exists for exactly one job: let
- * `agenticreplay compare --models a,b,c` reach several models without re-typing a gateway URL and a key on
- * every invocation.
- *
- * That means this file holds a credential, and everything here follows from it. The file is
- * `0600` inside a `0700` directory, the key can live in an environment variable instead of on
- * disk, and nothing ever prints it back. The key itself never enters a trace either — the proxy
- * adds it to the outbound request only, while what gets recorded is derived from the *incoming*
- * request with auth stripped, so a gateway key agenticreplay injects is invisible to the recording by
- * construction rather than by a rule someone has to remember.
- */
-export interface GatewayConfig {
-  /** Origin that serves the model APIs. One gateway usually serves both wire formats. */
-  url: string;
-  /** Stored key. Prefer `api_key_env` if you would rather not keep a credential on disk. */
-  api_key?: string;
-  /** Name of an environment variable to read the key from at call time. */
-  api_key_env?: string;
-  /** Explicit selection provenance; implicit destinations are never eligible for upload. */
-  url_source?: 'named' | 'default';
-}
-
-export interface AgenticReplayConfig {
-  gateway?: GatewayConfig;
-  /** Default model list for `agenticreplay compare` when `--models` is not given. */
-  models?: string[];
-}
-
-/** XDG, with the documented fallback. Honours `XDG_CONFIG_HOME` so tests need no real home. */
-export function configPath(env: NodeJS.ProcessEnv = process.env): string {
-  const base = env.XDG_CONFIG_HOME ?? join(env.HOME ?? homedir(), '.config');
-  return join(base, 'agenticreplay', 'config.json');
-}
-
-/**
- * Read the config, or an empty one.
- *
- * Never throws. A hand-edited file with a stray comma should cost you the gateway setting, not
- * every agenticreplay command — and a tool people reach for when something is already broken is the worst
- * possible thing to have its own unrecoverable failure mode.
- */
-export async function readConfig(
-  env: NodeJS.ProcessEnv = process.env,
-): Promise<AgenticReplayConfig> {
-  try {
-    const raw = await readFile(configPath(env), 'utf8');
-    const parsed: unknown = JSON.parse(raw);
-    return parsed !== null && typeof parsed === 'object' ? (parsed as AgenticReplayConfig) : {};
-  } catch {
-    return {};
-  }
-}
-
-export async function writeConfig(
-  config: AgenticReplayConfig,
-  env: NodeJS.ProcessEnv = process.env,
-): Promise<string> {
-  const path = configPath(env);
-  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
-  // Set explicitly as well as passed to mkdir: an existing directory keeps whatever mode it had,
-  // and a 0755 directory makes the file's 0600 decoration. On Windows neither mode argument does
-  // anything at all, so this is what keeps the directory from being listable by every account.
-  if (process.platform === 'win32') await restrictToOwner(dirname(path), 0o700);
-  else await restrictToOwner(dirname(path), 0o700).catch(() => {});
-
-  // Through a staging file, narrowed before the key is put into it.
-  //
-  // `mode: 0o600` is discarded on Windows, so a file written straight to `path` carries whatever
-  // the directory handed down until `restrictToOwner` has run on it — and if that call throws
-  // (icacls unavailable, `SystemRoot` unset, an unwritable `%TEMP%` for its scratch file) the key
-  // is already on disk and readable, while `agenticreplay setup` reports an error that mentions none of
-  // that. Measured: the key was there in plain text, under the inherited ACL.
-  //
-  // Neither truncating in place nor deleting on failure would do, because both throw away the
-  // config that was already there when the restriction fails. A rename carries the ACL of the
-  // file being moved, so the key is never on disk under any ACL but the intended one, and a
-  // failure leaves the previous config untouched.
-  // Named per writer, not per destination. `${path}.incoming` is one scratch file every caller
-  // addresses, with nothing serialising them: B's `writeFile(staging, '')` truncates the file A is
-  // about to rename into place, and A installs an empty `config.json` over the user's own while
-  // reporting success — `readConfig` swallows the parse failure, so the gateway and its key are
-  // simply gone. `BlobStore.put` and scrub's `commit` both randomise for this reason; sync uses
-  // the deterministic name only while holding a lock.
-  const staging = `${path}.${randomBytes(6).toString('hex')}.incoming`;
-  await writeFile(staging, '', { mode: 0o600, flag: 'wx' });
-  try {
-    await restrictToOwner(staging, 0o600);
-    await writeFile(staging, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 });
-    await rename(staging, path);
-  } catch (err) {
-    await rm(staging, { force: true }).catch(() => undefined);
-    throw err;
-  }
-  return path;
-}
-
-/**
- * The auth header for the configured gateway, or nothing.
- *
- * Returns `{}` rather than an empty bearer when no key is available. An `Authorization: Bearer `
- * with nothing after it is worse than sending none: a gateway may accept it as an anonymous
- * session, and the user never learns their key was not applied.
- */
-export function gatewayHeaders(
-  config: AgenticReplayConfig,
-  env: NodeJS.ProcessEnv = process.env,
-): Record<string, string> {
-  const gateway = config.gateway;
-  if (!gateway) return {};
-  const key = gateway.api_key ?? (gateway.api_key_env ? env[gateway.api_key_env] : undefined) ?? '';
-  if (key === '') return {};
-  // Both header names, because a gateway fronting both wire formats reads whichever its callers
-  // send — and sending an extra header costs nothing while guessing wrong costs a confusing 401.
-  return { authorization: `Bearer ${key}`, 'x-api-key': key };
-}
-
-/**
- * Two URLs that name the same host, for deciding whether a stored credential may travel.
- *
- * Lives here rather than beside either caller because both callers are credential gates —
- * `upstreamPlan` for model traffic and `resolveGateway` for push/pull — and two comparators that
- * drift apart mean one of them starts leaking. Falls back to a trimmed string compare when either
- * side does not parse, which errs towards withholding: an unparseable URL matches only itself.
- */
-export function sameOrigin(a: string, b: string): boolean {
-  try {
-    return new URL(a).origin === new URL(b).origin;
-  } catch {
-    return a.replace(/\/+$/, '') === b.replace(/\/+$/, '');
-  }
-}
-
-/**
- * Where live model calls go: flag, then environment, then the configured gateway.
- *
- * Needed by record *and* by replay — `--loose` and any fork continue live, and a fork that ignored
- * the override would quietly talk to the real provider instead of the gateway the user pointed it
- * at. Same precedence, whichever command is running: the more specific and more recent the
- * instruction, the more it wins.
- */
-export async function resolveUpstream(
-  args: ParsedArgs,
-  env: NodeJS.ProcessEnv = process.env,
-): Promise<Record<string, string> | undefined> {
-  const config = await readConfig(env);
-  const gateway = config.gateway?.url;
-  const out: Record<string, string> = {};
-  const anthropic =
-    args.str('upstream-anthropic') ?? env.AGENTICREPLAY_UPSTREAM_ANTHROPIC ?? gateway;
-  const openai = args.str('upstream-openai') ?? env.AGENTICREPLAY_UPSTREAM_OPENAI ?? gateway;
-  if (anthropic) out.anthropic = anthropic;
-  if (openai) {
-    out.openai = openai;
-    // The proxy resolves an origin by dialect id, and chat completions and the Responses API are
-    // two dialects sharing one provider — so a map holding only `openai` sends every Codex and
-    // Agents-SDK call straight past a gateway the user configured, on their own key, silently.
-    // `--upstream-openai` names a provider, not a wire format.
-    out['openai-responses'] = openai;
-  }
-  return Object.keys(out).length > 0 ? out : undefined;
-}
-
-/** A destination explicitly configured in AgenticReplay's own configuration namespace. */
-export function namedPushDestination(config: AgenticReplayConfig): string | undefined {
-  const url = config.gateway?.url;
-  if (!url) return undefined;
-  const source = config.gateway?.url_source;
-  if (source === 'named') return url;
-  if (source === 'default') return undefined;
-  return url;
-}

package/packages/cli/src/exchange-events.ts

@@ -1,369 +0,0 @@
-import { createHash } from 'node:crypto';
-import type {
-  CanonicalContent,
-  CanonicalRequest,
-  CanonicalResponse,
-} from '@agenticreplay/plugin-api';
-import type { RecordedExchange } from '@agenticreplay/proxy';
-import type { TraceWriter } from '@agenticreplay/core';
-
-/**
- * Turning intercepted model exchanges into trace events.
- *
- * This is where the project's central observation is cashed in. A tool result never appears in the
- * response that requested it — the harness runs the tool and hands the result back on the *next*
- * request. So we hold each `tool_use` open and close it when a later request carries the matching
- * `tool_result`, which is how a proxy that only sees model traffic still reconstructs the whole
- * tool loop without touching the agent.
- *
- * The consequence to keep in mind: a tool call whose result never comes back (the run ended, the
- * agent crashed) stays unresolved, and `unresolved()` reports it rather than silently dropping it.
- */
-
-export interface PendingToolCall {
-  id: string;
-  name: string;
-  input: unknown;
-  /** seq of the tool.call event, so the tool.result can point back at it. */
-  seq: number;
-  turn: number;
-}
-
-export interface DerivedEvent {
-  type: 'model.request' | 'model.response' | 'tool.call' | 'tool.result' | 'retrieval.context';
-  actor: 'agent' | 'model' | 'harness';
-  attrs: Record<string, unknown>;
-  payload?: unknown;
-  /** Index into the pending map, resolved to a real seq by the caller. */
-  causesToolId?: string;
-  /**
-   * Positions of earlier events *in this same batch* that caused this one.
-   *
-   * Seqs do not exist until the writer has appended, so the deriver cannot name them. It names
-   * positions in the array it is about to return instead, and the recorder swaps each one for the
-   * seq it wrote. Both edges this carries are observed rather than inferred: a `tool_use` block is
-   * inside the response that emitted it, and a `tool_result` block is inside the request that
-   * carried it back.
-   */
-  causesIndex?: number[];
-}
-
-export class ExchangeEventDeriver {
-  readonly #pending = new Map<string, PendingToolCall>();
-  /**
-   * Tool results already turned into events. Necessary because the conversation is resent in
-   * full every turn: without this, a result would be re-emitted on every subsequent request and
-   * a long run would accumulate one duplicate per turn per tool call.
-   */
-  readonly #closed = new Set<string>();
-  /**
-   * Retrieval contexts already emitted, by fingerprint.
-   *
-   * A conversation is resent in full every turn, so without this a multi-turn RAG session would
-   * re-emit the first turn's passages on every request after it — the same duplication `#closed`
-   * prevents for tool results, one event type over.
-   */
-  readonly #seenContext = new Set<string>();
-
-  /** Tool calls that were issued but whose result never came back. */
-  unresolved(): PendingToolCall[] {
-    // A pending entry now outlives the result that answered it, so that the recorder can still
-    // resolve the call's seq for `causes` after derive has returned. `#closed` is the authority on
-    // whether a call was actually answered; "still in #pending" no longer means "unanswered".
-    return [...this.#pending.values()].filter((p) => !this.#closed.has(p.id));
-  }
-
-  /**
-   * Derive the events for one exchange. Returns them in the order they should be appended;
-   * `tool.result` events come first because they describe work that happened *before* this
-   * request was made.
-   */
-  derive(exchange: RecordedExchange, turn: number): DerivedEvent[] {
-    const events: DerivedEvent[] = [];
-    /** Where the results this batch produced ended up, so the request can name them. */
-    const resultsAt: number[] = [];
-
-    // What the retriever found, before the request that carried it — same order and the same
-    // reason as a tool result: it describes work that happened before this call was made, and the
-    // request names it as a cause. Derived rather than captured, because it is already on the
-    // wire: the passages are in the prompt. See `retrievalContext`.
-    const retrieved = retrievalContext(exchange.canonicalRequest);
-    if (retrieved && !this.#seenContext.has(retrieved.fingerprint)) {
-      this.#seenContext.add(retrieved.fingerprint);
-      events.push({
-        type: 'retrieval.context',
-        // The harness retrieved these; agenticreplay only read them out of what it sent.
-        actor: 'harness',
-        attrs: {
-          query: retrieved.query,
-          passages: retrieved.passages.length,
-          chars: retrieved.passages.reduce((n, p) => n + p.length, 0),
-          // Named so a reader knows the boundaries are the prompt's, not the retriever's: this
-          // is what was *used*, which `--context-docs` truncates out of what was *retrieved*.
-          derived_from: 'prompt',
-        },
-        payload: { query: retrieved.query, passages: retrieved.passages },
-      });
-      resultsAt.push(events.length - 1);
-    }
-
-    for (const result of collectToolResults(exchange.canonicalRequest)) {
-      // Already accounted for on an earlier turn — the resent conversation is not new information.
-      if (this.#closed.has(result.tool_use_id)) continue;
-      // No pending call either: recording began mid-conversation, so this result is the first
-      // thing we know about that tool use. Record it rather than lose it.
-      const pending = this.#pending.get(result.tool_use_id);
-      events.push({
-        type: 'tool.result',
-        actor: 'harness',
-        attrs: {
-          tool_use_id: result.tool_use_id,
-          name: pending?.name ?? 'unknown',
-          is_error: result.is_error ?? false,
-          bytes: result.content.length,
-        },
-        payload: result.content,
-        causesToolId: result.tool_use_id,
-      });
-      resultsAt.push(events.length - 1);
-      // Deliberately not deleted here. The recorder resolves the call's seq *after* derive
-      // returns — it has to, since the seq only exists once the event is written — so dropping the
-      // entry in this loop meant `causes` came back empty on every tool result ever recorded, with
-      // no error to notice. `#closed` is what stops a resent result being re-emitted, so the
-      // pending entry can outlive the result it answered without any duplication.
-      this.#closed.add(result.tool_use_id);
-    }
-
-    events.push({
-      type: 'model.request',
-      actor: 'agent',
-      attrs: {
-        ...(exchange.requestId ? { request_id: exchange.requestId } : {}),
-        model: exchange.canonicalRequest.model,
-        dialect: exchange.dialect,
-        path: exchange.path,
-        messages: exchange.canonicalRequest.messages.length,
-        tools: exchange.canonicalRequest.tools?.length ?? 0,
-        // Only where agenticreplay established the connection and therefore knows. Absent reads as "not
-        // recorded" rather than as HTTP/1.1, which is why it is left off instead of defaulted.
-        ...(exchange.alpn === undefined ? {} : { alpn: exchange.alpn }),
-      },
-      payload: exchange.rawRequest,
-      // Absent rather than empty when nothing came back: an edge list naming nothing is noise a
-      // reader has to interpret, and `causes` is optional precisely so it can be left off.
-      ...(resultsAt.length > 0 ? { causesIndex: resultsAt } : {}),
-    });
-    const requestAt = events.length - 1;
-
-    const response = exchange.canonicalResponse;
-    events.push({
-      // The edge every other request/response pair already had. Without it `agenticreplay graph --to` on a
-      // model.response walks backwards, finds no incoming edge, and reports that the run has no
-      // causal edges at all -- on a run whose graph is otherwise full of them. `net.response`,
-      // `tool.result` and `shell.result` all name what they answered; this is the same fact about
-      // the pair the recorder watched most directly.
-      causesIndex: [requestAt],
-      type: 'model.response',
-      actor: 'model',
-      attrs: {
-        ...(exchange.requestId ? { request_id: exchange.requestId } : {}),
-        model: response?.model ?? exchange.canonicalRequest.model,
-        stop_reason: response?.stop_reason ?? 'unknown',
-        input_tokens: response?.usage.input_tokens ?? 0,
-        output_tokens: response?.usage.output_tokens ?? 0,
-        status: exchange.status,
-        duration_ms: exchange.durationMs ?? 0,
-        streamed: exchange.streamed,
-        // The payload below is the decoded body, so the encoding it arrived in is recorded here or
-        // nowhere: a promoted exchange keeps no response headers.
-        ...(exchange.responseDecodedFrom === undefined
-          ? {}
-          : { decoded_from: exchange.responseDecodedFrom }),
-        // Who answered. The trace said what was sent and what came back and never this, which is
-        // the first question asked of a recording that looks wrong — a gateway left behind in
-        // `~/.agenticreplay/config.json` redirects every run on the machine, and nothing in the run said so.
-        // On the response because it is a fact about the answer, and per exchange rather than in
-        // the manifest because the origin is chosen per request: a fork that changes provider
-        // changes it again mid-run. Absent means an older agenticreplay did not record it, never "the
-        // vendor's own API".
-        ...(exchange.upstream === undefined ? {} : { upstream: exchange.upstream }),
-      },
-      payload: exchange.rawResponse,
-    });
-    const responseAt = events.length - 1;
-
-    for (const use of collectToolUses(response)) {
-      events.push({
-        type: 'tool.call',
-        actor: 'model',
-        attrs: { tool_use_id: use.id, name: use.name, input: use.input },
-        causesIndex: [responseAt],
-      });
-      // seq is assigned by the writer; the caller patches it back in via markPending.
-      this.#pending.set(use.id, { id: use.id, name: use.name, input: use.input, seq: -1, turn });
-    }
-
-    return events;
-  }
-
-  markPending(toolUseId: string, seq: number): void {
-    const p = this.#pending.get(toolUseId);
-    if (p) p.seq = seq;
-  }
-
-  seqOf(toolUseId: string): number | undefined {
-    const seq = this.#pending.get(toolUseId)?.seq;
-    return seq === undefined || seq < 0 ? undefined : seq;
-  }
-}
-
-/**
- * The question a retriever was asked and the passages it came back with, read out of the prompt.
- *
- * The whole point is that this needs no new capture mechanism. A RAG system's retrieval runs
- * in-process — FAISS is a library call, not an HTTP request — so a proxy cannot see it happen.
- * What a proxy *can* see is the prompt the retrieval produced, and the passages are in it
- * verbatim, because putting them there is what retrieval is for. Same standing as `tool.call`,
- * which is likewise reconstructed from traffic rather than captured at the source.
- *
- * Two things are deliberately not here, and their absence is not an oversight. **Scores** and
- * **top-k** never reach the wire: a prompt carries the passages that survived truncation, in the
- * order they were pasted, with no numbers attached. Reporting a count as though it were `top-k`
- * would be a fabricated measurement — `--context-docs` cuts the list before it is joined, so even
- * "how many were retrieved" cannot be recovered. Getting those needs an instrument inside the
- * Python process, which is a separate piece of work and honestly out of reach from here.
- *
- * Recognition is deliberately narrow. It fires on an explicit `Context:` … `Question:` structure,
- * which is what a RAG template writes and what an ordinary chat turn does not. A looser rule —
- * "the message is long and has paragraphs" — would file half the coding-agent traffic in the
- * world as retrieval evidence, and a timeline that cries retrieval on every long prompt is worth
- * less than no timeline at all.
- */
-const CONTEXT_LABEL =
-  /^[ \t]*(?:context|retrieved context|relevant context|retrieved documents|documents|passages|sources)[ \t]*:[ \t]*/im;
-const QUESTION_LABEL = /^[ \t]*(?:question|query|user question)[ \t]*:[ \t]*/im;
-/** How a template separates one passage from the next, most explicit first. */
-const PASSAGE_SEPARATORS = [/\n[ \t]*-{3,}[ \t]*\n/, /\n[ \t]*={3,}[ \t]*\n/, /\n\s*\n/];
-
-export interface RetrievalContext {
-  query: string;
-  passages: string[];
-  /** Identity for de-duplication across resent conversations. */
-  fingerprint: string;
-}
-
-export function retrievalContext(req: CanonicalRequest): RetrievalContext | undefined {
-  const last = req.messages[req.messages.length - 1];
-  if (!last || last.role !== 'user') return undefined;
-  const text = last.content
-    .filter((b): b is Extract<CanonicalContent, { type: 'text' }> => b.type === 'text')
-    .map((b) => b.text)
-    .join('\n');
-  if (text === '') return undefined;
-
-  const context = CONTEXT_LABEL.exec(text);
-  if (context === null) return undefined;
-  const afterContext = context.index + context[0].length;
-  const question = QUESTION_LABEL.exec(text.slice(afterContext));
-  if (question === null) return undefined;
-
-  const body = text.slice(afterContext, afterContext + question.index).trim();
-  if (body === '') return undefined;
-
-  // What follows the question label, up to a blank line — a template's closing instruction
-  // ("Answer concisely:") sits after one, and folding it into the query would make two runs that
-  // asked the same thing look like they asked different things.
-  const rest = text.slice(afterContext + question.index + question[0].length);
-  const query = (rest.split(/\n\s*\n/)[0] ?? '').trim();
-  if (query === '') return undefined;
-
-  const separator = PASSAGE_SEPARATORS.find((re) => re.test(body));
-  // `String.split` with a non-global pattern still splits on every occurrence, so the separator
-  // is used as found rather than recompiled.
-  const passages = (separator === undefined ? [body] : body.split(separator))
-    .map((p) => p.trim())
-    .filter((p) => p !== '');
-  if (passages.length === 0) return undefined;
-
-  return {
-    query,
-    passages,
-    fingerprint: createHash('sha256')
-      .update(`${query}${passages.join('')}`)
-      .digest('hex'),
-  };
-}
-
-function collectToolResults(
-  req: CanonicalRequest,
-): Array<Extract<CanonicalContent, { type: 'tool_result' }>> {
-  const out: Array<Extract<CanonicalContent, { type: 'tool_result' }>> = [];
-  // Only the trailing user message can carry results for the calls we are still holding open;
-  // scanning the whole conversation would re-emit every earlier result on every turn.
-  const last = req.messages[req.messages.length - 1];
-  if (!last || last.role !== 'user') return out;
-  for (const block of last.content) {
-    if (block.type === 'tool_result') out.push(block);
-  }
-  return out;
-}
-
-function collectToolUses(
-  res: CanonicalResponse | undefined,
-): Array<Extract<CanonicalContent, { type: 'tool_use' }>> {
-  if (!res) return [];
-  return res.content.filter(
-    (b): b is Extract<CanonicalContent, { type: 'tool_use' }> => b.type === 'tool_use',
-  );
-}
-
-/**
- * Derive one exchange's events, resolve their causal references, and write them.
- *
- * Shared because it has two callers — `agenticreplay record` and the live half of a fork — and they drifted:
- * the fork wrote every model exchange with no `causes` at all, so a forked run could be replayed
- * and forked again but not explained. The seq resolution is the part that is easy to leave out,
- * since it cannot happen until the writer has appended, so it lives here rather than in either
- * caller.
- */
-export async function appendDerivedEvents(
-  writer: TraceWriter,
-  deriver: ExchangeEventDeriver,
-  exchange: RecordedExchange,
-  turn: number,
-  /**
-   * Each derived event as it is written.
-   *
-   * The recorder needs to know whether the agent ran any commands, so it can say when the shell
-   * layer was on and saw none of them. That is only knowable from what the model asked for, and
-   * this is the one place the derived events exist.
-   */
-  onDerived?: (derived: DerivedEvent) => void,
-): Promise<void> {
-  /** `written[i]` is the seq of the i-th derived event, which is what `causesIndex` refers to. */
-  const written: number[] = [];
-  for (const derived of deriver.derive(exchange, turn)) {
-    const causes: number[] = [];
-    if (derived.causesToolId) {
-      const seq = deriver.seqOf(derived.causesToolId);
-      if (seq !== undefined) causes.push(seq);
-    }
-    for (const index of derived.causesIndex ?? []) {
-      const seq = written[index];
-      if (seq !== undefined) causes.push(seq);
-    }
-    const event = await writer.append({
-      type: derived.type,
-      actor: derived.actor,
-      turn,
-      attrs: derived.attrs,
-      payload: derived.payload as never,
-      ...(causes.length > 0 ? { causes } : {}),
-    });
-    written.push(event.seq);
-    onDerived?.(derived);
-    if (derived.type === 'tool.call') {
-      deriver.markPending(String(derived.attrs['tool_use_id']), event.seq);
-    }
-  }
-}