@data-fair/lib-agents-sim 0.2.0

package/README.md

@@ -0,0 +1,165 @@
+# @data-fair/lib-agents-sim
+
+Primitives for judged browser simulations of an [agents](https://github.com/data-fair/agents)
+chat: drive a real conversation with a simulated user in a real browser, capture
+what happened, and hand the transcript to a judge. It does not own cases, login,
+settings seeding, or the turn loop — those stay in the host repo, which knows
+its own routes, fixtures and account setup. It also ships a small Claude Code
+bridge (a dev tool, not a simulation primitive) that exposes the Claude Agent
+SDK as an OpenAI-compatible provider, so a dev workspace can drive the assistant
+under test on a Claude subscription instead of a metered API key.
+
+## Install
+
+```bash
+npm i -D @data-fair/lib-agents-sim
+```
+
+## Which half needs which peer
+
+Every peer is **optional**, because the package has two independent halves and
+few consumers want both. Install only what the half you use needs.
+
+| You want | Install |
+| --- | --- |
+| The **harness** primitives (`createChatDriver`, `captureGateway`, `selectCases`, `reportCases`, `writeEvidence`, …) | `@playwright/test` |
+| The **bridge** (`df-agents-bridge`) | `@anthropic-ai/claude-agent-sdk` and `@modelcontextprotocol/sdk` |
+
+```bash
+# harness only
+npm i -D @playwright/test
+# bridge only
+npm i -D @anthropic-ai/claude-agent-sdk @modelcontextprotocol/sdk
+```
+
+The harness primitives are built against Playwright's `Page` / `FrameLocator`
+types and run inside your own Playwright project, so bring your own version.
+The bridge needs nothing from Playwright, which is why a bridge-only consumer
+is not forced into a browser install.
+
+Run `df-agents-bridge` without the two SDKs and it exits with an actionable
+message instead of a raw `ERR_MODULE_NOT_FOUND`, naming the install command
+above.
+
+**zod warning.** If your tree also contains the `ai` package, installing the
+Agent SDK may hoist zod 4 and break `ai`'s type inference. Add
+`"overrides": { "@anthropic-ai/claude-agent-sdk": { "zod": "3.25.76" } }`.
+
+## `df-agents-sim-init`
+
+```bash
+npx df-agents-sim-init [--force]
+```
+
+Copies the `/simulate` skill and the `simulation-judge` sub-agent definition
+into your repo's `.claude/skills/simulate/SKILL.md` and
+`.claude/agents/simulation-judge.md`. These cannot be loaded from
+`node_modules` — Claude Code reads them from the repository — so they are
+copied, not referenced, and **can drift** from the version in this package.
+The command prints the package version it copied from so drift is at least
+detectable; without `--force` it skips a file that already exists rather than
+overwriting local edits.
+
+## Minimal runner example
+
+A host repo owns the Playwright test that drives one case end to end. For a
+chat embedded in an iframe (the common case for a host application), pass
+`page.frameLocator('iframe')` to `createChatDriver`; for a page where the chat
+*is* the page, pass `page` itself — the driver's selectors are identical
+either way.
+
+```ts
+import { test } from '@playwright/test'
+import {
+  createChatDriver, captureGateway, nextUserMessage, isDone,
+  writeEvidence, selectCases, type Transcript, type SimulationCase
+} from '@data-fair/lib-agents-sim'
+
+const cases: SimulationCase[] = [
+  { name: 'find-a-dataset', route: '/embed/chat', persona: 'A curious analyst.', goal: 'Find last quarter\'s sales dataset.', maxTurns: 6 }
+]
+
+for (const simCase of selectCases(cases, [])) {
+  test(`simulation: ${simCase.name}`, async ({ page }) => {
+    const gateway = captureGateway(page)
+    await page.goto(simCase.route)
+
+    const chat = createChatDriver(page.frameLocator('iframe'))
+    const conversation: Array<{ role: string, text: string }> = []
+    let error: string | undefined
+
+    try {
+      for (let i = 0; i < simCase.maxTurns; i++) {
+        const message = await nextUserMessage(simCase, conversation, simCase.maxTurns - i)
+        if (isDone(message)) break
+        await chat.sendMessage(message)
+        await chat.waitForTurn()
+        // Read into a local FIRST, then replace: clearing up front means a throw
+        // from readConversation leaves the transcript empty, losing every prior
+        // turn — and an empty transcript is the one thing a judge cannot judge.
+        const read = await chat.readConversation()
+        conversation.length = 0
+        conversation.push(...read)
+      }
+    } catch (err) {
+      error = err instanceof Error ? err.message : String(err)
+    }
+
+    const transcript: Transcript = { case: simCase.name, goal: simCase.goal, persona: simCase.persona, route: simCase.route, conversation, gateway, consoleErrors: [] }
+    // `valid` is derived, never hardcoded: the sidecar exists to tell a run that
+    // really happened apart from one that fell over, so that `reportCases` says
+    // "invalid (…)" instead of re-reporting the previous run's verdict.
+    writeEvidence(simCase.name, transcript, {
+      case: simCase.name, valid: !error, error,
+      assistantModel: 'sonnet', userModel: 'haiku',
+      turns: conversation.length / 2, durationMs: 0, finishedAt: new Date().toISOString()
+    })
+    if (error) throw new Error(`run invalid: ${error}`)
+  })
+}
+```
+
+Then judge each written transcript with the `simulation-judge` sub-agent (via
+the copied `/simulate` skill), and turn the evidence directory into a pass/fail
+summary with `reportCases(cases, evidenceDir)` — the host repo's own report
+script decides where cases live and what to do with the failure count it
+returns.
+
+### Where the evidence goes
+
+`writeEvidence(name, transcript, sidecar, dir?)` writes `sim-<name>.json` (the
+transcript the judge reads) and `sim-<name>.run.json` (the validity sidecar).
+`dir` defaults to the exported `evidenceDir`, which is
+`path.join(process.cwd(), 'simulations', 'tmp')` — resolved against the host
+repo's working directory, so the default only makes sense if you run your suite
+from the repository root. Pass `dir` explicitly to put evidence anywhere else,
+and hand the same directory to `reportCases(cases, dir)` so the reader and the
+writer agree.
+
+## Scripts the copied `/simulate` skill expects
+
+`df-agents-sim-init` copies the skill **verbatim**, and the skill refers to npm
+scripts by the names the origin repository uses. It cannot know yours, so define
+these three in your `package.json` (adjust the paths to your layout):
+
+```json
+{
+  "scripts": {
+    "dev-bridge": "df-agents-bridge",
+    "simulate": "playwright test -c playwright.sim.config.ts --project=simulate",
+    "simulate:report": "node simulations/report.ts"
+  }
+}
+```
+
+- `dev-bridge` — starts the bridge the simulated user and the assistant both
+  talk to. Must be running before `simulate`.
+- `simulate` — runs your Playwright project containing the scenario specs. Keep
+  it out of your default `test` script: a bare `npm test` would otherwise spend
+  plan quota.
+- `simulate:report` — calls `reportCases(cases, evidenceDir)` and exits non-zero
+  on the failure count it returns.
+
+If you prefer different names, edit the copied
+`.claude/skills/simulate/SKILL.md` to match — but remember that a later
+`df-agents-sim-init --force` overwrites it.

package/bin/init.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/bin/init.js

@@ -0,0 +1,28 @@
+#!/usr/bin/env node
+/**
+ * Copies the judge definition and the /simulate skill into the consuming repo's
+ * .claude/ directory. They cannot be loaded from node_modules — Claude Code reads
+ * them from the repository — so they are copied and can drift. The version is
+ * printed so drift is at least detectable.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+const here = path.dirname(fileURLToPath(import.meta.url));
+const templates = path.join(here, '..', 'templates');
+const pkg = JSON.parse(fs.readFileSync(path.join(here, '..', 'package.json'), 'utf8'));
+const cwd = process.cwd();
+const targets = [
+    { from: 'simulation-judge.md', to: path.join(cwd, '.claude', 'agents', 'simulation-judge.md') },
+    { from: 'simulate-skill.md', to: path.join(cwd, '.claude', 'skills', 'simulate', 'SKILL.md') }
+];
+for (const { from, to } of targets) {
+    fs.mkdirSync(path.dirname(to), { recursive: true });
+    if (fs.existsSync(to) && !process.argv.includes('--force')) {
+        console.log(`skipped (exists, use --force): ${path.relative(cwd, to)}`);
+        continue;
+    }
+    fs.copyFileSync(path.join(templates, from), to);
+    console.log(`wrote ${path.relative(cwd, to)}`);
+}
+console.log(`from @data-fair/lib-agents-sim@${pkg.version}`);

package/bridge/conversation.d.ts

@@ -0,0 +1,68 @@
+import { type OpenAIToolCall } from './openai.ts';
+export declare const ABORTED_TOOL_RESULT = "ERROR: the conversation was aborted before this tool call could be answered. No result is available.";
+export type TurnOutcome = {
+    type: 'tools';
+    calls: OpenAIToolCall[];
+} | {
+    type: 'done';
+    usage?: object;
+} | {
+    type: 'error';
+    message: string;
+};
+export type SdkMessage = {
+    type: string;
+    subtype?: string;
+    message?: {
+        content?: Array<{
+            type: string;
+            text?: string;
+        }>;
+    };
+    usage?: {
+        input_tokens?: number;
+        output_tokens?: number;
+    };
+    result?: unknown;
+};
+export declare class Conversation {
+    #private;
+    key: string;
+    lastSeen: number;
+    pending: Map<string, (result: string) => void>;
+    /**
+     * The tool names the live MCP tool server was built with. A continuation
+     * request declaring a different set cannot be served by this query — the model
+     * would be offered the stale set — so the server compares before adopting it.
+     */
+    toolNames: string[];
+    constructor(key: string, toolNames?: string[]);
+    /** Handed to the SDK as `options.abortController`, so abort() really cancels the query. */
+    get controller(): AbortController;
+    get signal(): AbortSignal;
+    /**
+     * True once the upstream query has ended (done, error or abort). A dead
+     * conversation must never be adopted as a continuation: nothing would ever
+     * resolve the turn it was handed.
+     */
+    get isDead(): boolean;
+    abort(): void;
+    /**
+     * Called by the MCP tool server. Records the call and suspends: the promise is
+     * resolved by deliverToolResults when the client's next request arrives.
+     */
+    handleToolCall(name: string, args: Record<string, unknown>): Promise<string>;
+    deliverToolResults(results: Array<{
+        id: string;
+        content: string;
+    }>): void;
+    /** True when every tool call handed back has been answered by this request. */
+    awaits(ids: string[]): boolean;
+    /**
+     * Attach an HTTP response as the sink and wait for this turn to end.
+     * Start the consumer loop on first use.
+     */
+    beginTurn(sink: (text: string) => void, iterator?: AsyncIterable<SdkMessage>): Promise<TurnOutcome>;
+    /** Mark the calls just handed back, so the next turn only reports new ones. */
+    handedBack(count: number): void;
+}

package/bridge/conversation.js

@@ -0,0 +1,167 @@
+/**
+ * One live SDK query, spanning many HTTP requests.
+ *
+ * The tricky part, and why this is not driven straight off the iterator: when the
+ * model calls a tool, the SDK yields the assistant message and THEN invokes the
+ * MCP handler. A consumer that checked for tool calls inside its `for await` body
+ * would look before the handler had run, loop, and block forever on an iterator
+ * the SDK is no longer feeding — the chat would hang on its first tool call.
+ *
+ * So the consumer loop runs for the whole conversation, writing into whichever
+ * HTTP response is currently attached, and a turn ends on whichever comes first:
+ * the handlers suspending (tool calls to hand back) or the query finishing.
+ */
+import crypto from 'node:crypto';
+import { mcpNameToTool, mapUsage } from "./openai.js";
+// Parallel tool calls in one assistant message arrive as separate handler
+// invocations a tick apart. Wait this long after the first before closing the
+// turn, so they are handed back together as OpenAI expects.
+const SETTLE_MS = 50;
+// What a suspended tool handler returns when the conversation is aborted. It
+// MUST read as a failure: the model receives it as the tool's result, and a
+// bare word like "aborted" would be taken for a successful answer.
+export const ABORTED_TOOL_RESULT = 'ERROR: the conversation was aborted before this tool call could be answered. No result is available.';
+export class Conversation {
+    key;
+    lastSeen = Date.now();
+    pending = new Map();
+    /**
+     * The tool names the live MCP tool server was built with. A continuation
+     * request declaring a different set cannot be served by this query — the model
+     * would be offered the stale set — so the server compares before adopting it.
+     */
+    toolNames;
+    #collected = [];
+    #handedBack = 0;
+    #sink = null;
+    #resolveTurn = null;
+    #settle = null;
+    #controller = new AbortController();
+    #consuming = false;
+    #terminal = null;
+    constructor(key, toolNames = []) {
+        this.key = key;
+        this.toolNames = toolNames;
+    }
+    /** Handed to the SDK as `options.abortController`, so abort() really cancels the query. */
+    get controller() { return this.#controller; }
+    get signal() { return this.#controller.signal; }
+    /**
+     * True once the upstream query has ended (done, error or abort). A dead
+     * conversation must never be adopted as a continuation: nothing would ever
+     * resolve the turn it was handed.
+     */
+    get isDead() { return this.#terminal !== null; }
+    abort() {
+        this.#terminal ??= { type: 'error', message: 'conversation aborted' };
+        this.#controller.abort();
+        // Unblock anything still waiting on a tool result.
+        for (const resolve of this.pending.values())
+            resolve(ABORTED_TOOL_RESULT);
+        this.pending.clear();
+    }
+    /**
+     * Called by the MCP tool server. Records the call and suspends: the promise is
+     * resolved by deliverToolResults when the client's next request arrives.
+     */
+    handleToolCall(name, args) {
+        const id = `call_${crypto.randomUUID()}`;
+        this.#collected.push({
+            id,
+            type: 'function',
+            function: { name: mcpNameToTool(name), arguments: JSON.stringify(args) }
+        });
+        if (this.#settle)
+            clearTimeout(this.#settle);
+        this.#settle = setTimeout(() => { this.#endTurn({ type: 'tools', calls: this.#collected.slice(this.#handedBack) }); }, SETTLE_MS);
+        if (this.#settle.unref)
+            this.#settle.unref();
+        return new Promise(resolve => { this.pending.set(id, resolve); });
+    }
+    deliverToolResults(results) {
+        for (const r of results) {
+            const resolve = this.pending.get(r.id);
+            if (!resolve)
+                continue;
+            this.pending.delete(r.id);
+            resolve(r.content);
+        }
+    }
+    /** True when every tool call handed back has been answered by this request. */
+    awaits(ids) {
+        if (this.#terminal)
+            return false;
+        return ids.length > 0 && ids.every(id => this.pending.has(id));
+    }
+    /**
+     * Attach an HTTP response as the sink and wait for this turn to end.
+     * Start the consumer loop on first use.
+     */
+    beginTurn(sink, iterator) {
+        if (this.#terminal) {
+            // The query ended between turns (rate limit, crash) with no response
+            // attached, so its outcome was recorded rather than dropped. Replay it:
+            // without this the request would await a promise nothing can settle.
+            return Promise.resolve(this.#terminal);
+        }
+        this.#sink = sink;
+        const turn = new Promise(resolve => { this.#resolveTurn = resolve; });
+        if (iterator && !this.#consuming) {
+            this.#consuming = true;
+            // #consume reports every failure through #endTurn, so nothing escapes here.
+            this.#consume(iterator).catch(() => { });
+        }
+        // A tool call can land after the previous turn's settle window closed: #endTurn
+        // was then a no-op and no timer stays armed, so the call would sit here forever
+        // while the SDK waits for a tool_result. Hand any such stragglers back now.
+        // A still-armed timer is left alone: it is about to close the turn properly,
+        // with the parallel siblings it is waiting for.
+        if (!this.#settle && this.#collected.length > this.#handedBack) {
+            this.#endTurn({ type: 'tools', calls: this.#collected.slice(this.#handedBack) });
+        }
+        return turn;
+    }
+    /** Mark the calls just handed back, so the next turn only reports new ones. */
+    handedBack(count) {
+        this.#handedBack += count;
+    }
+    async #consume(iterator) {
+        let usage;
+        try {
+            for await (const msg of iterator) {
+                if (msg.type === 'assistant') {
+                    for (const block of msg.message?.content ?? []) {
+                        if (block.type === 'text' && block.text)
+                            this.#sink?.(block.text);
+                    }
+                }
+                if (msg.type === 'result') {
+                    usage = mapUsage(msg.usage);
+                    if (msg.subtype !== 'success') {
+                        this.#endTurn({ type: 'error', message: String(msg.result ?? msg.subtype) });
+                        return;
+                    }
+                }
+            }
+            this.#endTurn({ type: 'done', usage });
+        }
+        catch (err) {
+            this.#endTurn({ type: 'error', message: err instanceof Error ? err.message : String(err) });
+        }
+    }
+    #endTurn(outcome) {
+        if (this.#settle) {
+            clearTimeout(this.#settle);
+            this.#settle = null;
+        }
+        const resolve = this.#resolveTurn;
+        this.#resolveTurn = null;
+        this.#sink = null;
+        // Anything but a hand-back means the query is over. Remember it: between two
+        // HTTP requests there is no resolver, and dropping the outcome here is what
+        // left the next request awaiting a promise that could never settle.
+        if (outcome.type !== 'tools')
+            this.#terminal ??= outcome;
+        resolve?.(outcome);
+    }
+}

package/bridge/index.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/bridge/index.js

@@ -0,0 +1,18 @@
+#!/usr/bin/env node
+import { MISSING_SDK_MESSAGE, isMissingSdkError } from "../missing-sdk.js";
+const port = Number(process.env.BRIDGE_PORT ?? 3194);
+try {
+    const { createServer } = await import("./server.js");
+    createServer({ port });
+    console.log(`claude-bridge listening on http://localhost:${port}`);
+    console.log('configure an "OpenAI Compatible" provider with this base URL + /v1 and Compatibility Mode = compatible');
+}
+catch (err) {
+    if (isMissingSdkError(err)) {
+        console.error(MISSING_SDK_MESSAGE);
+        process.exitCode = 1;
+    }
+    else {
+        throw err;
+    }
+}

package/bridge/openai.d.ts

@@ -0,0 +1,102 @@
+/**
+ * Pure translation between the OpenAI chat-completions wire format the agents
+ * gateway speaks and what the Claude Agent SDK accepts. No I/O, no SDK import —
+ * everything here is unit-tested.
+ */
+export type OpenAIToolCall = {
+    id: string;
+    type: 'function';
+    function: {
+        name: string;
+        arguments: string;
+    };
+};
+export type OpenAIMessage = {
+    role: 'system' | 'user' | 'assistant' | 'tool';
+    content?: string | null;
+    tool_calls?: OpenAIToolCall[];
+    tool_call_id?: string;
+};
+export type OpenAIToolDef = {
+    type: 'function';
+    function: {
+        name: string;
+        description?: string;
+        parameters?: Record<string, unknown>;
+    };
+};
+export declare const MCP_SERVER_NAME = "bridge";
+export declare function toolNameToMcp(name: string): string;
+export declare function mcpNameToTool(name: string): string;
+export declare function extractSystemPrompt(messages: OpenAIMessage[]): string;
+/**
+ * Render the conversation as a single prompt, for the replay path (a fresh
+ * session). The continuation path never comes through here — it delivers tool
+ * results into a query that is already holding the history.
+ */
+export declare function renderTranscript(messages: OpenAIMessage[]): string;
+export declare function textChunk(id: string, model: string, text: string): {
+    choices: {
+        index: number;
+        delta: {
+            content: string;
+        };
+        finish_reason: null;
+    }[];
+    id: string;
+    object: string;
+    created: number;
+    model: string;
+};
+export declare function toolCallsChunk(id: string, model: string, calls: OpenAIToolCall[]): {
+    choices: {
+        index: number;
+        delta: {
+            tool_calls: {
+                index: number;
+                id: string;
+                type: string;
+                function: {
+                    name: string;
+                    arguments: string;
+                };
+            }[];
+        };
+        finish_reason: null;
+    }[];
+    id: string;
+    object: string;
+    created: number;
+    model: string;
+};
+export declare function finalChunk(id: string, model: string, finishReason: 'stop' | 'tool_calls', usage?: object): {
+    usage?: object | undefined;
+    choices: {
+        index: number;
+        delta: {};
+        finish_reason: "stop" | "tool_calls";
+    }[];
+    id: string;
+    object: string;
+    created: number;
+    model: string;
+};
+/**
+ * The SDK also reports total_cost_usd, deliberately not forwarded: it is list-price
+ * bookkeeping, while consumption is actually against the plan's rate limits. Showing
+ * it in the usage UI would invite reading it as real spend.
+ */
+export declare function mapUsage(usage: {
+    input_tokens?: number;
+    output_tokens?: number;
+} | undefined): {
+    prompt_tokens: number;
+    completion_tokens: number;
+    total_tokens: number;
+} | undefined;
+export declare function errorBody(message: string, type?: string): {
+    error: {
+        message: string;
+        type: string;
+    };
+};

package/bridge/openai.js

@@ -0,0 +1,91 @@
+/**
+ * Pure translation between the OpenAI chat-completions wire format the agents
+ * gateway speaks and what the Claude Agent SDK accepts. No I/O, no SDK import —
+ * everything here is unit-tested.
+ */
+// Tools reach the model namespaced by their MCP server, so every name makes a
+// round trip through this prefix and must be mapped back before it is emitted.
+export const MCP_SERVER_NAME = 'bridge';
+const PREFIX = `mcp__${MCP_SERVER_NAME}__`;
+export function toolNameToMcp(name) {
+    return PREFIX + name;
+}
+export function mcpNameToTool(name) {
+    return name.startsWith(PREFIX) ? name.slice(PREFIX.length) : name;
+}
+export function extractSystemPrompt(messages) {
+    return messages
+        .filter(m => m.role === 'system')
+        .map(m => m.content ?? '')
+        .join('\n\n');
+}
+/**
+ * Render the conversation as a single prompt, for the replay path (a fresh
+ * session). The continuation path never comes through here — it delivers tool
+ * results into a query that is already holding the history.
+ */
+export function renderTranscript(messages) {
+    const lines = ['<conversation_history>'];
+    for (const m of messages) {
+        if (m.role === 'system')
+            continue;
+        if (m.role === 'tool') {
+            lines.push(`tool result (id=${m.tool_call_id}): ${m.content ?? ''}`);
+            continue;
+        }
+        if (m.role === 'assistant' && m.tool_calls?.length) {
+            for (const c of m.tool_calls) {
+                lines.push(`assistant called tool (id=${c.id}) ${c.function.name} with ${c.function.arguments}`);
+            }
+            if (m.content)
+                lines.push(`assistant: ${m.content}`);
+            continue;
+        }
+        lines.push(`${m.role}: ${m.content ?? ''}`);
+    }
+    lines.push('</conversation_history>');
+    lines.push('');
+    lines.push('Continue this conversation: produce the next assistant turn.');
+    return lines.join('\n');
+}
+const base = (id, model) => ({
+    id,
+    object: 'chat.completion.chunk',
+    created: Math.floor(Date.now() / 1000),
+    model
+});
+export function textChunk(id, model, text) {
+    return { ...base(id, model), choices: [{ index: 0, delta: { content: text }, finish_reason: null }] };
+}
+export function toolCallsChunk(id, model, calls) {
+    return {
+        ...base(id, model),
+        choices: [{
+                index: 0,
+                delta: { tool_calls: calls.map((c, index) => ({ index, id: c.id, type: 'function', function: c.function })) },
+                finish_reason: null
+            }]
+    };
+}
+export function finalChunk(id, model, finishReason, usage) {
+    return {
+        ...base(id, model),
+        choices: [{ index: 0, delta: {}, finish_reason: finishReason }],
+        ...(usage ? { usage } : {})
+    };
+}
+/**
+ * The SDK also reports total_cost_usd, deliberately not forwarded: it is list-price
+ * bookkeeping, while consumption is actually against the plan's rate limits. Showing
+ * it in the usage UI would invite reading it as real spend.
+ */
+export function mapUsage(usage) {
+    if (!usage)
+        return undefined;
+    const prompt = usage.input_tokens ?? 0;
+    const completion = usage.output_tokens ?? 0;
+    return { prompt_tokens: prompt, completion_tokens: completion, total_tokens: prompt + completion };
+}
+export function errorBody(message, type = 'api_error') {
+    return { error: { message, type } };
+}