@data-fair/lib-agents-sim 0.4.1 → 0.6.0

package/chat-driver.d.ts

@@ -24,11 +24,23 @@ export declare function chatDriverStrings(locale: ChatDriverLocale): {
 };
 export declare const TURN_TIMEOUT_MS: number;
 export declare const SEND_TIMEOUT_MS = 15000;
+/**
+ * How a turn ended. `waiting` means the assistant declared
+ * `wait_for_user_action` and is holding the turn open for the person — the
+ * caller's cue to let them act, then wait again for the turn it resumes.
+ */
+export type TurnOutcome = 'ended' | 'waiting';
+/**
+ * Matched on the activity's kind, not its label: the label is the model's own
+ * words interpolated into a translated string, so any text match would be both
+ * locale-dependent and at the mercy of what the assistant wrote.
+ */
+export declare const WAITING_SELECTOR = "[data-testid=\"chat-activity\"][data-activity=\"waiting\"]";
 export declare function createChatDriver(root: ChatRoot, opts?: {
     locale?: ChatDriverLocale;
 }): {
     sendMessage(text: string): Promise<void>;
-    waitForTurn(timeoutMs?: number): Promise<void>;
+    waitForTurn(timeoutMs?: number): Promise<TurnOutcome>;
     readConversation(): Promise<{
         role: "user" | "assistant";
         text: string;

package/chat-driver.js

@@ -46,6 +46,12 @@ export const TURN_TIMEOUT_MS = 10 * 60 * 1000;
 // later with no diagnosis. Unrelated to TURN_TIMEOUT_MS, which bounds a
 // legitimately long model turn once the message has actually been sent.
 export const SEND_TIMEOUT_MS = 15000;
+/**
+ * Matched on the activity's kind, not its label: the label is the model's own
+ * words interpolated into a translated string, so any text match would be both
+ * locale-dependent and at the mercy of what the assistant wrote.
+ */
+export const WAITING_SELECTOR = '[data-testid="chat-activity"][data-activity="waiting"]';
 export function createChatDriver(root, opts = {}) {
     const strings = chatDriverStrings(opts.locale ?? 'en');
     return {
@@ -77,10 +83,23 @@ export function createChatDriver(root, opts = {}) {
         },
         async waitForTurn(timeoutMs = TURN_TIMEOUT_MS) {
             const stop = root.getByRole('button', { name: strings.stop });
+            const waiting = root.locator(WAITING_SELECTOR);
             // The turn may already be finished by the time we look, so a missing Stop
             // button is not an error — only one that never goes away is.
             await stop.waitFor({ state: 'visible', timeout: 15000 }).catch(() => { });
-            await expect(stop).toHaveCount(0, { timeout: timeoutMs });
+            // A turn can finish two ways, and only one of them is the assistant being
+            // done. `wait_for_user_action` holds the turn open on purpose, having handed
+            // control back to the person — and a simulated person only acts between
+            // turns, so a harness that waited for the Stop button alone could never let
+            // them act on it. Every declared wait then ran its whole window and was
+            // recorded as a wedged turn; at a wait window as long as the harness's own
+            // ceiling, that is every run.
+            const ended = expect(stop).toHaveCount(0, { timeout: timeoutMs }).then(() => 'ended');
+            const armed = expect(waiting).toHaveCount(1, { timeout: timeoutMs }).then(() => 'waiting', 
+            // Never rejects: a wait that is simply not what this turn did must not be
+            // the error a caller sees. The Stop arm owns the timeout message.
+            () => new Promise(() => { }));
+            return await Promise.race([ended, armed]);
         },
         async readConversation() {
             // evaluateAll, not page.evaluate: FrameLocator has no evaluate, and this

package/gateway-capture.d.ts

@@ -22,10 +22,58 @@ export type GatewayExchange = {
         name: string;
         arguments: string;
     }>;
+    /** Characters the application injected as `<host-state>` / `<host-events>` /
+     * `<hidden-context>` blocks, across every message of this request. They land in
+     * tool results and hidden context, neither of which survives into this summary,
+     * so the size of what the host reports can only be measured here. Cumulative
+     * like the history, so the last request of a conversation carries its total.
+     */
+    hostBlockChars: number;
+    /** What each tool ANSWERED, paired with the call that asked. Cumulative like
+     * `toolCalls`. Without these a judge sees only what the assistant asked a tool
+     * and can never check a claim about the answer against it — "the form data is
+     * valid and saved" was unfalsifiable in a recorded run for exactly this reason.
+     * Each result is capped: one large payload must not swallow the evidence file.
+     */
+    toolResults: Array<{
+        id: string;
+        name: string;
+        result: string;
+    }>;
     /** When true, this exchange was received but postData could not be parsed. Evidence of
      * a failed capture rather than an absent request.
      */
     unparsed?: true;
 };
+export declare const TOOL_RESULT_MAX_CHARS = 2000;
+/** Only closed blocks count: an unterminated open tag would otherwise swallow
+ *  the whole rest of the message and report it as host overhead. */
+export declare function countHostBlockChars(messages: unknown[]): number;
+type Body = {
+    model?: string;
+    messages?: Array<{
+        role?: string;
+        content?: unknown;
+        tool_call_id?: string;
+        tool_calls?: Array<{
+            id?: string;
+            function?: {
+                name?: string;
+                arguments?: string;
+            };
+        }>;
+    }>;
+    tools?: Array<{
+        function?: {
+            name?: string;
+        };
+    }>;
+};
+export declare function extractToolResults(messages: NonNullable<Body['messages']>): {
+    id: string;
+    name: string;
+    result: string;
+}[];
 export declare function summariseRequest(body: unknown): GatewayExchange | null;
 export declare function captureGateway(page: Page): GatewayExchange[];
+export {};

package/gateway-capture.js

@@ -1,3 +1,53 @@
+export const TOOL_RESULT_MAX_CHARS = 2000;
+const HOST_BLOCKS = ['host-state', 'host-events', 'hidden-context'];
+/** Only closed blocks count: an unterminated open tag would otherwise swallow
+ *  the whole rest of the message and report it as host overhead. */
+export function countHostBlockChars(messages) {
+    let total = 0;
+    for (const message of messages) {
+        const content = message?.content;
+        const text = typeof content === 'string' ? content : JSON.stringify(content ?? '');
+        for (const name of HOST_BLOCKS) {
+            const open = `<${name}>`;
+            const close = `</${name}>`;
+            let from = 0;
+            for (;;) {
+                const start = text.indexOf(open, from);
+                if (start === -1)
+                    break;
+                const end = text.indexOf(close, start);
+                if (end === -1)
+                    break;
+                total += end + close.length - start;
+                from = end + close.length;
+            }
+        }
+    }
+    return total;
+}
+function renderToolResult(content) {
+    const text = typeof content === 'string' ? content : JSON.stringify(content ?? '');
+    return text.length <= TOOL_RESULT_MAX_CHARS ? text : text.slice(0, TOOL_RESULT_MAX_CHARS) + '…[truncated]';
+}
+export function extractToolResults(messages) {
+    // The call that asked may live in any earlier message, so the name is looked
+    // up by id across the whole history rather than by adjacency.
+    const names = new Map();
+    for (const m of messages) {
+        for (const call of m.tool_calls ?? []) {
+            if (call.id)
+                names.set(call.id, call.function?.name ?? '');
+        }
+    }
+    return messages
+        .filter(m => m.role === 'tool')
+        .map(m => ({
+        id: m.tool_call_id ?? '',
+        // An unpaired result is still evidence; recording it nameless beats dropping it.
+        name: names.get(m.tool_call_id ?? '') ?? '',
+        result: renderToolResult(m.content)
+    }));
+}
 function extractUserMessageText(content) {
     if (typeof content === 'string')
         return content;
@@ -29,6 +79,8 @@ export function summariseRequest(body) {
         toolNames: (b.tools ?? []).map(t => t.function?.name ?? '').filter(Boolean),
         messageCount: b.messages.length,
         lastUserMessage: extractUserMessageText(last?.content),
+        hostBlockChars: countHostBlockChars(b.messages),
+        toolResults: extractToolResults(b.messages),
         toolCalls: b.messages.flatMap(m => (m.tool_calls ?? []).map(c => ({
             name: c.function?.name ?? '',
             arguments: c.function?.arguments ?? ''
@@ -52,6 +104,8 @@ export function captureGateway(page) {
                 messageCount: 0,
                 lastUserMessage: '',
                 toolCalls: [],
+                hostBlockChars: 0,
+                toolResults: [],
                 unparsed: true
             });
             return;

package/index.d.ts

@@ -1,10 +1,11 @@
 export type { SimulationCase, Transcript, RunSidecar } from './types.ts';
 export { createNeutralCwd, scrubEnv, isolationOptions, type Env } from './isolation.ts';
 export { captureGateway, summariseRequest, type GatewayExchange } from './gateway-capture.ts';
-export { nextUserMessage, personaSystemPrompt, personaPrompt, isDone, DONE, PERSONA_MAX_TURNS, PERCEPTION_INSTRUCTIONS } from './persona.ts';
+export { nextUserMessage, personaSystemPrompt, personaPrompt, isDone, DONE, PERSONA_MAX_TURNS, PERCEPTION_INSTRUCTIONS, DEFAULT_USER_MODEL, resolveUserModel } from './persona.ts';
 export { writeEvidence, evidenceDir } from './transcript.ts';
+export { computeMetrics, type RunMetrics } from './metrics.ts';
 export { selectCases } from './cases.ts';
 export { reportCases } from './report.ts';
-export { createChatDriver, chatDriverStrings, type ChatRoot, type ChatDriverLocale, TURN_TIMEOUT_MS, SEND_TIMEOUT_MS } from './chat-driver.ts';
+export { createChatDriver, chatDriverStrings, type ChatRoot, type ChatDriverLocale, type TurnOutcome, TURN_TIMEOUT_MS, SEND_TIMEOUT_MS, WAITING_SELECTOR } from './chat-driver.ts';
 export { createPagePerception, truncate, SNAPSHOT_CAP, ACTION_TIMEOUT_MS, MCP_SERVER_NAME as PAGE_MCP_SERVER_NAME } from './page-perception.ts';
 export type { PerceptionRoot, Observation, PagePerception } from './page-perception.ts';

package/index.js

@@ -1,8 +1,9 @@
 export { createNeutralCwd, scrubEnv, isolationOptions } from "./isolation.js";
 export { captureGateway, summariseRequest } from "./gateway-capture.js";
-export { nextUserMessage, personaSystemPrompt, personaPrompt, isDone, DONE, PERSONA_MAX_TURNS, PERCEPTION_INSTRUCTIONS } from "./persona.js";
+export { nextUserMessage, personaSystemPrompt, personaPrompt, isDone, DONE, PERSONA_MAX_TURNS, PERCEPTION_INSTRUCTIONS, DEFAULT_USER_MODEL, resolveUserModel } from "./persona.js";
 export { writeEvidence, evidenceDir } from "./transcript.js";
+export { computeMetrics } from "./metrics.js";
 export { selectCases } from "./cases.js";
 export { reportCases } from "./report.js";
-export { createChatDriver, chatDriverStrings, TURN_TIMEOUT_MS, SEND_TIMEOUT_MS } from "./chat-driver.js";
+export { createChatDriver, chatDriverStrings, TURN_TIMEOUT_MS, SEND_TIMEOUT_MS, WAITING_SELECTOR } from "./chat-driver.js";
 export { createPagePerception, truncate, SNAPSHOT_CAP, ACTION_TIMEOUT_MS, MCP_SERVER_NAME as PAGE_MCP_SERVER_NAME } from "./page-perception.js";

package/metrics.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Facts derived from a recorded run, handed to the judge as evidence.
+ *
+ * Nothing here scores anything. There is no threshold, no enum, no pass mark —
+ * a metric that decided whether a run was good would narrow the judge's
+ * attention to the things that happen to be countable, which is the opposite of
+ * what the judge is for. These exist because a few quantities are tedious and
+ * error-prone to count by reading a transcript, and two of them are outright
+ * traps: `toolCalls` is cumulative, and a sub-agent's requests are interleaved
+ * with the lead's.
+ */
+import type { Transcript } from './types.ts';
+export type RunMetrics = {
+    /** Messages the person actually sent. */
+    userMessages: number;
+    assistantBubbles: number;
+    /**
+     * Bubbles that rendered no prose. Almost always a tool-call bubble, which DOES
+     * render a chip naming the tool on screen — so this is not a count of silence,
+     * and three judges in a row misread it as one. What it measures is how much of
+     * a run the person watched as tool names rather than sentences.
+     */
+    textlessAssistantBubbles: number;
+    avgVisibleReplyChars: number | null;
+    /** Requests to the gateway, every conversation included. */
+    modelRequests: number;
+    /** The role that served the person's own conversation (the first request's). */
+    leadModel: string;
+    leadRequests: number;
+    nonLeadRequests: number;
+    /**
+     * Requests per model role, when the record names one. This is the authority:
+     * inferring roles from message counts reported three `summarizer` compaction
+     * calls as sub-agent dispatches in a run that made no sub-agent call at all.
+     * Null when no request carries a role, where the interleaving split is all
+     * there is.
+     */
+    requestsByModel: Record<string, number> | null;
+    /** `modelRequests / userMessages`, to one decimal; null when nobody spoke. */
+    requestsPerUserMessage: number | null;
+    /** The biggest prompt handed to a role other than the lead's, and which role
+     *  took it; null when every request was the lead's. A summarizer legitimately
+     *  carries the conversation, so read the role before reading the size. */
+    largestNonLeadPromptChars: number | null;
+    largestNonLeadPromptModel: string | null;
+    /** Tool calls issued twice with identical arguments, across every conversation. */
+    duplicateToolCalls: number;
+    /** Characters the host injected as `<host-state>` / `<host-events>` blocks;
+     *  null for a run recorded before the capture measured them. */
+    hostBlockChars: number | null;
+};
+export declare function computeMetrics(transcript: Transcript): RunMetrics;

package/metrics.js

@@ -0,0 +1,95 @@
+const signature = (e) => JSON.stringify([...e.toolNames].sort());
+/**
+ * Split the gateway record back into the conversations that produced it.
+ *
+ * One conversation's `messageCount` only ever grows; a sub-agent starts a fresh
+ * one at 2 while the lead is already deep, and the lead then resumes. The tool
+ * set is a hint rather than the rule, because page tools come and go with the
+ * route — a recorded lead went 14 to 19 to 26 tools without ever restarting.
+ */
+function splitConversations(gateway) {
+    // `streams` is kept in most-recently-used order, so `opened` is what restores
+    // the order the conversations actually started in — which is what makes the
+    // first one the lead's.
+    const streams = [];
+    for (const exchange of gateway) {
+        // Most recently used first: when the tool set gives no answer, the lead is
+        // the stream that spoke last.
+        const open = streams.filter(s => s.last < exchange.messageCount).reverse();
+        const target = open.find(s => s.tools === signature(exchange)) ?? open[0];
+        if (target) {
+            target.exchanges.push(exchange);
+            target.last = exchange.messageCount;
+            target.tools = signature(exchange);
+            // Move to the back, so "most recently used" stays true.
+            streams.splice(streams.indexOf(target), 1);
+            streams.push(target);
+        }
+        else {
+            streams.push({ exchanges: [exchange], last: exchange.messageCount, tools: signature(exchange), opened: streams.length });
+        }
+    }
+    return [...streams].sort((a, b) => a.opened - b.opened).map(s => s.exchanges);
+}
+/** Within one conversation the tool-call list is cumulative, so the longest is the whole of it. */
+function finalToolCalls(stream) {
+    let longest = [];
+    for (const e of stream)
+        if (e.toolCalls.length > longest.length)
+            longest = e.toolCalls;
+    return longest;
+}
+function countDuplicates(calls) {
+    const seen = new Set(calls.map(c => JSON.stringify([c.name, c.arguments])));
+    return calls.length - seen.size;
+}
+export function computeMetrics(transcript) {
+    const userMessages = transcript.conversation.filter(m => m.role === 'user').length;
+    const assistant = transcript.conversation.filter(m => m.role === 'assistant');
+    const visible = assistant.filter(m => m.text.trim().length > 0);
+    // Prefer what the record says over what the shape implies.
+    const roled = transcript.gateway.filter(e => e.model);
+    const leadModel = transcript.gateway[0]?.model ?? '';
+    const requestsByModel = roled.length
+        ? roled.reduce((acc, e) => { acc[e.model] = (acc[e.model] ?? 0) + 1; return acc; }, {})
+        : null;
+    const conversations = splitConversations(transcript.gateway);
+    let lead;
+    let nonLead;
+    if (requestsByModel && leadModel) {
+        lead = transcript.gateway.filter(e => e.model === leadModel);
+        nonLead = transcript.gateway.filter(e => e.model !== leadModel);
+    }
+    else {
+        // No role in the record: the first request answers the person's first
+        // message, so the conversation it opened is the lead's.
+        const [first = [], ...rest] = conversations;
+        lead = first;
+        nonLead = rest.flat();
+    }
+    const biggest = nonLead.reduce((best, e) => (!best || e.lastUserMessage.length > best.lastUserMessage.length ? e : best), null);
+    const measuredHostBlocks = transcript.gateway.some(e => typeof e.hostBlockChars === 'number');
+    return {
+        userMessages,
+        assistantBubbles: assistant.length,
+        textlessAssistantBubbles: assistant.length - visible.length,
+        avgVisibleReplyChars: visible.length
+            ? Math.round(visible.reduce((sum, m) => sum + m.text.trim().length, 0) / visible.length)
+            : null,
+        modelRequests: transcript.gateway.length,
+        leadModel,
+        leadRequests: lead.length,
+        nonLeadRequests: nonLead.length,
+        requestsByModel,
+        requestsPerUserMessage: userMessages
+            ? Math.round((transcript.gateway.length / userMessages) * 10) / 10
+            : null,
+        largestNonLeadPromptChars: biggest ? biggest.lastUserMessage.length : null,
+        largestNonLeadPromptModel: biggest ? (biggest.model || null) : null,
+        duplicateToolCalls: conversations.reduce((sum, s) => sum + countDuplicates(finalToolCalls(s)), 0),
+        hostBlockChars: measuredHostBlocks
+            // Cumulative like the history, so a conversation's last request holds its total.
+            ? conversations.reduce((sum, s) => sum + Math.max(0, ...s.map(e => e.hostBlockChars ?? 0)), 0)
+            : null
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@data-fair/lib-agents-sim",
-  "version": "0.4.1",
+  "version": "0.6.0",
   "description": "Primitives for judged browser simulations of the data-fair agents chat, plus a Claude Code bridge exposing the Agent SDK as an OpenAI-compatible provider.",
   "main": "index.js",
   "type": "module",

package/page-perception.d.ts

@@ -31,6 +31,15 @@ export type PagePerception = {
      */
     offLimits: string[];
 };
+/**
+ * Head AND tail, because overlays live at the end.
+ *
+ * A dialog is teleported to the end of the DOM, so a head-only cut removes
+ * precisely what a person is being asked to look at. A judged run turned on it:
+ * the persona clicked "Ajouter une colonne", its snapshot was cut at the same
+ * point before and after the click, and whether the dialog ever opened was not
+ * decidable from the record — the judge had to say so instead of ruling.
+ */
 export declare function truncate(text: string): string;
 export declare function createPagePerception(roots: PerceptionRoot[], opts?: {
     offLimits?: string[];

package/page-perception.js

@@ -34,8 +34,22 @@ export const SNAPSHOT_CAP = 4000;
 // must surface through the existing try/catch as a recorded observation
 // instead of an unrecorded hang.
 export const ACTION_TIMEOUT_MS = 15000;
+/**
+ * Head AND tail, because overlays live at the end.
+ *
+ * A dialog is teleported to the end of the DOM, so a head-only cut removes
+ * precisely what a person is being asked to look at. A judged run turned on it:
+ * the persona clicked "Ajouter une colonne", its snapshot was cut at the same
+ * point before and after the click, and whether the dialog ever opened was not
+ * decidable from the record — the judge had to say so instead of ruling.
+ */
 export function truncate(text) {
-    return text.length <= SNAPSHOT_CAP ? text : text.slice(0, SNAPSHOT_CAP) + '…[truncated]';
+    if (text.length <= SNAPSHOT_CAP)
+        return text;
+    const marker = '\n…[truncated]\n';
+    const head = Math.floor(SNAPSHOT_CAP * 0.6);
+    const tail = SNAPSHOT_CAP - head;
+    return text.slice(0, head) + marker + text.slice(-tail);
 }
 const TOOLS = [
     {
@@ -93,19 +107,38 @@ export function createPagePerception(roots, opts = {}) {
         }
         return null;
     };
+    /** As above, but says WHICH finder matched — `click` reports it. */
+    const firstMatchKind = async (finders) => {
+        for (const [kind, find] of finders) {
+            try {
+                const loc = find();
+                if (await loc.count() > 0)
+                    return { kind, loc };
+            }
+            catch { /* a finder that throws simply does not match */ }
+        }
+        return null;
+    };
     const click = async (name) => {
         if (isOffLimits(name))
             return OFF_LIMITS_RESULT;
         for (const { root } of roots) {
-            const loc = await firstMatch([
-                () => root.getByRole('button', { name }).first(),
-                () => root.getByRole('link', { name }).first(),
-                () => root.getByText(name).first()
+            const match = await firstMatchKind([
+                ['control', () => root.getByRole('button', { name }).first()],
+                ['control', () => root.getByRole('link', { name }).first()],
+                ['text', () => root.getByText(name).first()]
             ]);
-            if (loc) {
+            if (match) {
                 try {
-                    await loc.click({ timeout: ACTION_TIMEOUT_MS });
-                    return `clicked "${name}"`;
+                    await match.loc.click({ timeout: ACTION_TIMEOUT_MS });
+                    // Playwright clicks whatever is visible, so the text fallback succeeds
+                    // on a paragraph as readily as on a button. Saying which one it was is
+                    // the difference between a person learning nothing happened and a
+                    // person concluding the product is broken — a recorded run did exactly
+                    // that, and the judge filed it as a product failure.
+                    return match.kind === 'control'
+                        ? `clicked "${name}"`
+                        : `clicked the text "${name}", which is not a button or a link — nothing may happen`;
                 }
                 catch (err) {
                     return `could not click "${name}": ${err instanceof Error ? err.message : String(err)}`;

package/persona.d.ts

@@ -1,6 +1,26 @@
 import type { SimulationCase } from './types.ts';
 import type { PagePerception } from './page-perception.ts';
 export declare const DONE = "DONE";
+/**
+ * The model the simulated person runs on unless the caller overrides it.
+ *
+ * Exported because hosts record which model ran, in a sidecar whose whole point
+ * is that verdicts from different tiers are never compared silently — and a host
+ * that re-derived this with its own literal recorded `haiku` for a run that
+ * actually used `sonnet`, the moment this default changed. One value, read by
+ * both the runner and the recorder.
+ *
+ * Sonnet, not haiku: on haiku the persona stopped enforcing its own goal —
+ * accepting a chat-only answer for a goal that demanded the result on screen,
+ * and asserting it could see nothing but the chat while its own look had just
+ * returned the page. A persona that lets the product off the hook produces green
+ * runs that prove nothing.
+ */
+export declare const DEFAULT_USER_MODEL = "sonnet";
+/** What `nextUserMessage` will actually run on, given an environment. */
+export declare function resolveUserModel(env?: {
+    SIM_USER_MODEL?: string;
+}): string;
 export declare const PERSONA_MAX_TURNS = 25;
 export declare const PERCEPTION_INSTRUCTIONS = "You can look at the screen yourself with the look tool, and you can click and type\non the page. Before you say anything about what is or is not on the screen, look.\nNever claim you cannot see something you have not looked for.";
 export declare function isDone(message: string): boolean;

package/persona.js

@@ -11,6 +11,26 @@ import { createNeutralCwd, isolationOptions } from "./isolation.js";
 import { MISSING_SDK_MESSAGE, isMissingSdkError } from "./missing-sdk.js";
 import { MCP_SERVER_NAME } from "./page-perception.js";
 export const DONE = 'DONE';
+/**
+ * The model the simulated person runs on unless the caller overrides it.
+ *
+ * Exported because hosts record which model ran, in a sidecar whose whole point
+ * is that verdicts from different tiers are never compared silently — and a host
+ * that re-derived this with its own literal recorded `haiku` for a run that
+ * actually used `sonnet`, the moment this default changed. One value, read by
+ * both the runner and the recorder.
+ *
+ * Sonnet, not haiku: on haiku the persona stopped enforcing its own goal —
+ * accepting a chat-only answer for a goal that demanded the result on screen,
+ * and asserting it could see nothing but the chat while its own look had just
+ * returned the page. A persona that lets the product off the hook produces green
+ * runs that prove nothing.
+ */
+export const DEFAULT_USER_MODEL = 'sonnet';
+/** What `nextUserMessage` will actually run on, given an environment. */
+export function resolveUserModel(env = process.env) {
+    return env.SIM_USER_MODEL || DEFAULT_USER_MODEL;
+}
 let neutralCwd;
 // The persona looks and acts before replying, so one turn is not enough:
 // look → act → look → reply, with room to spare.
@@ -47,8 +67,15 @@ reply with your message; the runner types and sends it for you.`;
 export function isDone(message) {
     if (!message)
         return false;
-    // Normalize the message: trim, strip quotes/backticks, strip trailing punctuation, uppercase
-    let normalized = message.trim();
+    // The LAST line, not the whole message. The persona is asked for DONE and
+    // nothing else, and a capable one still signs off first ("That matches what
+    // I'm seeing — good.\n\nDONE"). Strict equality on the whole message missed
+    // that, so the runner sent the sign-off to the assistant and paid a full model
+    // request for a pleasantry nobody reads — once per case, every suite. It still
+    // has to be a line of its own, or "let me know when it is DONE" would end the
+    // run on the person's own words.
+    const lines = message.trim().split('\n').map(l => l.trim()).filter(Boolean);
+    let normalized = lines[lines.length - 1] ?? '';
     // Strip surrounding quotes or backticks
     if ((normalized.startsWith('"') && normalized.endsWith('"')) ||
         (normalized.startsWith("'") && normalized.endsWith("'")) ||
@@ -124,7 +151,7 @@ export async function nextUserMessage(c, conversation, turnsLeft, opts) {
         prompt: personaPrompt(conversation, turnsLeft),
         options: {
             ...isolationOptions(neutralCwd),
-            model: process.env.SIM_USER_MODEL ?? 'haiku',
+            model: resolveUserModel(),
             systemPrompt: personaSystemPrompt(c, !!opts?.perception, !!opts?.perception?.offLimits.length),
             // Unconditional: a caller with no perception registers no mcpServers, so
             // the persona has no tool to call and the loop still ends after the one
@@ -146,11 +173,18 @@ export async function nextUserMessage(c, conversation, turnsLeft, opts) {
                 : {})
         }
     })) {
+        // The LAST assistant message that carried text, not every one of them. With
+        // perception wired in the SDK emits an assistant message per reasoning step
+        // between tool calls, and appending them all sent the persona's inner
+        // monologue to the assistant as the person's own words — in one recorded run
+        // naming the tool the assistant should call, in another welding DONE onto the
+        // end of a sentence so isDone() missed it and the run ran on for an extra turn.
         if (msg.type === 'assistant') {
-            for (const block of msg.message?.content ?? []) {
-                if (block.type === 'text' && block.text)
-                    text += block.text;
-            }
+            const said = (msg.message?.content ?? [])
+                .filter((block) => block.type === 'text' && block.text)
+                .map((block) => block.text);
+            if (said.length)
+                text = said.join('');
         }
     }
     return text.trim();

package/report.js

@@ -4,6 +4,32 @@
  */
 import fs from 'node:fs';
 import path from 'node:path';
+/**
+ * The counted facts, printed as facts. No threshold turns any of these into a
+ * failure: what they cost is a judgement, and the judge makes it with the
+ * transcript in hand.
+ */
+function metricsLine(run) {
+    const m = run?.metrics;
+    if (!m)
+        return '';
+    const parts = [
+        run?.toolsModel ? `[${run.assistantModel} / tools ${run.toolsModel}]` : '',
+        `${m.modelRequests} model requests for ${m.userMessages} user messages`,
+        m.requestsPerUserMessage === null ? '' : `(${m.requestsPerUserMessage}/message)`,
+        m.requestsByModel
+            ? `(${Object.entries(m.requestsByModel).map(([role, n]) => `${role} ${n}`).join(', ')})`
+            : (m.nonLeadRequests ? `· ${m.nonLeadRequests} non-lead` : ''),
+        m.largestNonLeadPromptChars
+            ? `· largest ${m.largestNonLeadPromptModel ?? 'non-lead'} prompt ${m.largestNonLeadPromptChars} chars`
+            : '',
+        `· ${m.textlessAssistantBubbles}/${m.assistantBubbles} bubbles tool-chips only`,
+        m.avgVisibleReplyChars === null ? '' : `· avg reply ${m.avgVisibleReplyChars} chars`,
+        m.duplicateToolCalls ? `· ${m.duplicateToolCalls} repeated tool calls` : '',
+        m.hostBlockChars === null ? '' : `· ${m.hostBlockChars} chars of host blocks`
+    ];
+    return parts.filter(Boolean).join(' ');
+}
 // The verdict is written by a model and is untrusted by construction — a
 // well-known slip is a stringified boolean ("false" instead of false), which
 // would otherwise pass truthiness checks and silently report success.
@@ -56,12 +82,21 @@ export function reportCases(cases, evidenceDir) {
     for (const row of rows)
         console.log(row.map((cell, i) => cell.padEnd(widths[i])).join('  '));
     for (const c of cases) {
+        const run = read(`sim-${c.name}.run.json`);
         const verdict = read(`sim-${c.name}.verdict.json`);
-        if (!isValidVerdict(verdict, c.name) || verdict.frictions.length === 0)
+        const line = metricsLine(run);
+        const hasDetail = isValidVerdict(verdict, c.name) && (verdict.frictions.length > 0 || (verdict.findings?.length ?? 0) > 0);
+        if (!line && !hasDetail)
+            continue;
+        console.log(`\n${c.name}${isValidVerdict(verdict, c.name) ? `: ${verdict.summary}` : ''}`);
+        if (line)
+            console.log(`  ${line}`);
+        if (!isValidVerdict(verdict, c.name))
             continue;
-        console.log(`\n${c.name}: ${verdict.summary}`);
         for (const f of verdict.frictions)
             console.log(`  - turn ${f.turn}: ${f.what} → ${f.effect}`);
+        for (const f of verdict.findings ?? [])
+            console.log(`  · [${f.severity}] ${f.area}: ${f.what} (${f.evidence})`);
     }
     console.log(failures === 0 ? '\nall cases satisfied' : `\n${failures} case(s) need attention`);
     return failures;

package/templates/agents-sim-skill.md

@@ -45,9 +45,54 @@ Ask the user to start anything that is down. Never start or stop dev processes y
    SIM_CASES=air-quality npm run simulate        # one case
    ```
 
-   Models are pinned by `SIM_ASSISTANT_MODEL` (default `sonnet`) and
-   `SIM_USER_MODEL` (default `haiku`), and recorded per run, so verdicts from
-   different tiers are never compared silently.
+   Models are pinned by `SIM_ASSISTANT_MODEL` (default `sonnet`),
+   `SIM_TOOLS_MODEL` (default `haiku`, for sub-agents, compaction and the
+   moderation guard) and `SIM_USER_MODEL` (default `sonnet`), and recorded per
+   run, so verdicts from different tiers are never compared silently.
+
+   The persona is on `sonnet` deliberately. On `haiku` it stopped enforcing its
+   own goal: a case whose goal said the answer had to be shown on screen was
+   ended with a chat-only reply and marked done, and another persona asserted it
+   could see nothing but the chat while its own `look` had just returned the
+   page. A persona that lets the product off the hook produces green runs that
+   prove nothing. It is the most expensive knob here, so lower it deliberately,
+   not by default.
+
+   **Confirm it started.** A suite takes minutes, so you will want to background
+   it — and a run that never launched looks exactly like a run still going. Check
+   the runner's own log first, and the sidecar as the stronger signal:
+
+   ```bash
+   tail -5 sim.log                    # state-setup passing means it is going
+   ls simulations/tmp/*.run.json      # a file here means a case really started
+   ```
+
+   The sidecar is written before each case's body, but AFTER the login fixture,
+   which can take more than a minute on a cold page — so an empty
+   `simulations/tmp/` on its own is not proof of a stall. A log with no progress
+   at all is. Check the directory you are looking in, too: a backgrounded
+   `cd X && … &` runs the `cd` in a subshell, so a later `ls` reads whatever
+   directory you started from. One session read another repo's stale evidence
+   that way and called a dead run healthy.
+
+   **Wait on the process id, never on a text pattern.** `pgrep -f` matches full
+   command lines, including the command line of the waiter you are writing — so
+
+   ```bash
+   until ! pgrep -f "playwright.sim.config"; do sleep 15; done   # WRONG
+   ```
+
+   matches itself and waits forever, silently, producing nothing. One session lost
+   an hour and three quarters to exactly this, and then repeated it while trying to
+   fix it. Capture the pid and watch that instead:
+
+   ```bash
+   nohup npm run simulate > sim.log 2>&1 &
+   SIMPID=$!
+   while kill -0 "$SIMPID" 2>/dev/null; do sleep 20; done
+   ```
+
+   For the same reason, never `pkill -f` a pattern taken from your own script.
 
 4. **Ignore the runner's own account of how it went.** The transcript at
    `simulations/tmp/sim-<case>.json` is the evidence. A Playwright `passed` line
@@ -61,8 +106,16 @@ Ask the user to start anything that is down. Never start or stop dev processes y
 
    - the case name and its goal
    - the transcript path, `simulations/tmp/sim-<case>.json`
+   - the sidecar path, `simulations/tmp/sim-<case>.run.json`
    - ask for the JSON verdict its own definition specifies
 
+   **Dispatch them all the same way.** What to look at, what matters, what you
+   suspect is wrong this time — none of that goes in the dispatch. The judge's
+   own definition sets its mandate, and it reads `docs/architecture/` itself. A
+   briefing you write per run steers the verdict toward what you already
+   believed and makes two runs' verdicts incomparable. If a judge is looking in
+   the wrong place, fix its definition, not one dispatch.
+
 6. **Write each verdict** to `simulations/tmp/sim-<case>.verdict.json` as raw JSON.
    Strip any code fence the judge added. A malformed verdict reports as
    `not judged`, which is deliberate — check the file rather than being surprised.
@@ -73,14 +126,23 @@ Ask the user to start anything that is down. Never start or stop dev processes y
    npm run simulate:report
    ```
 
-   Relay the summary and the friction list. Exit code is non-zero if any case was
-   unsatisfactory, invalid, not judged, or never ran.
+   Relay the summary, the friction list and the findings. Exit code is non-zero
+   if any case was unsatisfactory, invalid, not judged, or never ran.
+
+8. **Root-cause what it found.** The judge reads a record, not the source: it
+   says a tool result misled the assistant, not which line built that result.
+   That half is yours, and it is where a run becomes a change. For each friction
+   point and each finding worth acting on, go into the code, find what produced
+   it, and say so — a defect you confirmed, a design decision the judge could
+   not see, or a harness artefact. Report that, not the verdict verbatim.
 
 ## Reading the result
 
-The friction list is the point. "Unsatisfactory" tells you a run went badly; a
-friction point names the reply or tool result that misled the person and what they
-did next — that is what turns a run into a concrete change.
+The friction list is the person's side, and it is what "unsatisfactory" is
+actually about: which reply or tool result misled them and what they did next.
+The findings are everything else the run exposed — cost, what the conversation
+was like to read, tools, the product, the harness. Both are claims about a
+record; both need confirming against the code before anyone acts on them.
 
 Rate limits are the practical ceiling: three Claude roles per case on one
 subscription. A run cut short by a rate limit is an **invalid run**, not a product

package/templates/simulation-judge.md

@@ -4,46 +4,82 @@ description: Judge one scenario simulation transcript and return a JSON verdict.
 tools: Read
 ---
 
-You are judging whether a chat assistant actually served a person, from the
-transcript of one simulated conversation.
+You are judging one simulated conversation between a person and a chat
+assistant, from the record the run left behind.
 
-You are given a case name, the person's goal, and a path to a transcript. Read
-the transcript with `Read`; do not ask for it to be pasted.
+Two questions, in this order. **Did the person get what they came for** — that
+is the one that decides `satisfied`, and nothing else does. Then: **what was
+wrong with how it went**, anywhere in what the record shows you. The second
+question is open. Cost and efficiency, the readability of what the person saw,
+tool descriptions and tool results, the product's own behaviour, the chat
+harness itself — all of it is in scope, and so is anything you notice that this
+list does not name. You are not filling in a form; you are the one reader who
+saw the whole run.
 
-The transcript holds:
-- `conversation` — what the person and the assistant said, as rendered on screen
-- `gateway` — every request the page made, carrying the tools it offered and the
-  tool calls the assistant actually made
-- `consoleErrors` — browser errors during the run
-- `observations` — what the person actually looked at and did, recorded per turn:
-  `{ turn, tool, args, result }`. `look` returns the accessibility outline of the
-  screen at that moment.
+## What you are given
+
+A case name, the person's goal, and paths to the evidence. Read the files with
+`Read`; do not ask for anything to be pasted.
+
+- `simulations/tmp/sim-<case>.json` — the transcript:
+  - `conversation` — what the person and the assistant said, as rendered on screen
+  - `gateway` — every request the page made, carrying the tools it offered, the
+    tool calls the assistant actually made, and in `toolResults` what each of
+    those calls answered. Use `toolResults` to check an assistant's claim against
+    what the tool really returned; large results are truncated, and both lists are
+    cumulative.
+  - `consoleErrors` — browser errors during the run
+  - `observations` — what the person actually looked at and did, per turn:
+    `{ turn, tool, args, result }`. `look` returns the accessibility outline of
+    the screen at that moment.
+- `simulations/tmp/sim-<case>.run.json` — the sidecar: which models ran, how long
+  it took, and `metrics`, a few counts derived from the transcript. Read them as
+  facts, not as a score: nothing in `metrics` is a pass mark, and a number only
+  means something once you have seen in the transcript what produced it.
+
+**Read the architecture documentation before judging.** `docs/architecture/` in
+this repository describes what the assistant is supposed to be, what the tools
+promise, and what the chat harness does on its own. Start with the document
+covering the agent or chat integration, then read whatever looks relevant to
+what you saw. Without it you will report intended design as a defect and miss
+the places the implementation diverges from what was written down.
+
+## How to read the record, and where it misleads
+
+`gateway` records what the browser SENT, and each request resends the whole
+conversation so far — so the assistant's FINAL reply never appears there,
+because no later request carries it. Read the last assistant turn from
+`conversation`. Never conclude "the assistant never answered" from `gateway`.
+
+`gateway[].toolCalls` is CUMULATIVE: exchange N holds every tool call from
+1..N. That is a history, not repetition. `metrics.duplicateToolCalls` already
+accounts for this; if you count repetition yourself, compare across exchanges
+first.
+
+Not every request is the assistant answering the person. Compaction, moderation
+and sub-agents each run on their own model role, with their own short history
+interleaved among the lead's. `metrics.requestsByModel` says which role served
+how many — read it before attributing spend, and never call a request a
+sub-agent dispatch unless a `subagent_*` call appears in `toolCalls`. A judge
+once reported three `summarizer` compaction calls as sub-agent work in a run
+that made no sub-agent call at all.
 
 A claim about what is on screen must be supported by a preceding `look` in
-`observations`. A persona asserting a visual fact it never observed is a HARNESS
-fault, not product friction — say so plainly in `notes` and do not count it as a
-friction point. This has happened: a run once had the person insist a panel was
-closed having never looked, and the judge reported it as a product failure.
-
-`gateway` records what the browser SENT to the server, and each request carries
-the whole conversation so far — so the assistant's FINAL reply of a conversation
-never appears there, because no later request resends it. Read the final
-assistant turn from `conversation`, not `gateway`, and never conclude "the
-assistant never answered" from its absence in `gateway`.
-
-`gateway[].toolCalls` is CUMULATIVE: exchange N contains every tool call from
-turns 1..N, not just that turn's. Do not treat this as the same tool call being
-repeated — compare call counts across exchanges before filing repetition as a
-friction point, or you will report calls that never actually recurred.
+`observations`. A persona asserting a visual fact it never observed is a
+HARNESS fault, not product friction — say so in `notes` and do not count it as
+friction. This has happened: a run had the person insist a panel was closed
+having never looked, and the judge reported it as a product failure.
+
+## `satisfied` and `frictions` — the person's side
 
 Judge the run against the goal, not against your idea of a good answer. The
-person is not a tester: if they had to ask three times, that is a finding even
-if the final answer was correct.
+person is not a tester: if they had to ask three times, that is friction even
+when the final answer was right.
+
+`satisfied` is true only if the goal was actually met and visibly so.
 
-**The friction list is the point.** A score says a run went badly; a friction
-point says which reply or tool result misled the person and what they concluded.
-That is what turns a run into a concrete change to a prompt or a tool
-description. Look especially for:
+A friction point names the reply or tool result that misled the person and what
+they did next. Look especially for:
 - the assistant claiming it did something the `gateway` record shows it never did
 - a tool offered but never used when it was obviously needed, or called with
   arguments that misread the person's words
@@ -51,6 +87,56 @@ description. Look especially for:
 - the person having to supply information the assistant could have looked up
 - an answer that is correct but never shown where the person asked for it
 
+A friction's `turn` is the 1-based index of the USER turn it occurred on — the
+Nth message the person sent, counting only their messages in `conversation`.
+A friction caused by the reply to the person's 3rd message is `"turn": 3`.
+Count this way so two judges reading the same transcript agree.
+
+An empty `frictions` array is a real answer when a run went cleanly.
+
+## `findings` — everything else the run exposed
+
+Separate from friction, because these are for whoever maintains the system
+rather than about what the person lived through. Anything the record supports:
+
+- **cost** — how much work the run took for what it delivered. `metrics` gives
+  you requests per user message, the split by model role, the largest prompt
+  handed to a role other than the lead's, and how many characters the
+  application injected into the conversation. A summarizer legitimately carries
+  the whole conversation, so read which role took a big prompt before calling
+  it waste. Then go look: which turns spent the requests,
+  and was the spending doing anything?
+- **conversation** — what the person had to read. Length, repetition, hedging,
+  restating what is already on their screen, silence where a word was owed.
+  `metrics.textlessAssistantBubbles` counts bubbles with no prose in them. These
+  are almost always tool-call bubbles, and the page DOES render a chip naming the
+  tool — so the count is not evidence of silence, and the `observations` outlines
+  will show you the chips. What it tells you is how much of the run the person
+  watched as tool names rather than sentences.
+- **tools** — a description that invites the wrong call, a result the model
+  visibly misread, a required argument the model had no way to know, an error
+  the assistant swallowed.
+- **product** — what the application did, or failed to do, behind the
+  conversation. `consoleErrors` and `observations` are the evidence.
+- **harness** — the chat itself: turn handling, streaming, the simulated
+  person's own tooling, anything in how the run was conducted rather than in
+  what was said.
+
+Severity is your call: `high` for something that cost the person the goal or
+would in a nearby run, `medium` for real waste or real confusion, `low` for
+things worth knowing. Evidence is a pointer a maintainer can check — a turn
+number, a tool name, a count, a quoted line.
+
+**Say what is wrong, not why the code does it.** You have the record, not the
+source. Naming a file or a function you did not read is a guess that sends
+someone down the wrong path; the agent that dispatched you has the codebase and
+does the root-causing. Describe the defect precisely enough for them to find it.
+
+Report nothing you cannot point at in the record. A short `findings` list from a
+clean run is worth more than a padded one.
+
+## Output
+
 Return ONLY raw JSON, no code fence, in exactly this shape:
 
 {
@@ -60,15 +146,11 @@ Return ONLY raw JSON, no code fence, in exactly this shape:
   "frictions": [
     { "turn": <number>, "what": "<what the assistant or a tool did>", "effect": "<what the person concluded or had to do>" }
   ],
-  "notes": "<anything a maintainer should know, or empty>"
+  "findings": [
+    { "area": "cost" | "conversation" | "tools" | "product" | "harness" | "<your own>", "severity": "high" | "medium" | "low", "what": "<the defect>", "evidence": "<where in the record>" }
+  ],
+  "notes": "<the overall opinion a maintainer should hear, or empty>"
 }
 
-`satisfied` is true only if the person's goal was actually met and visibly so.
-An empty `frictions` array is a real answer when a run went cleanly.
-
-A friction's `turn` is the 1-based index of the USER turn it occurred on — the
-Nth message the person sent, counting only the person's messages in
-`conversation` and not the assistant's. So a friction caused by the
-assistant's reply to the person's 3rd message is still `"turn": 3`. Count
-consistently this way so that two judges reading the same transcript would
-agree on the number.
+`notes` is where the judgement that fits no list goes: how the run read, what
+you would change first, what you are unsure about.

package/transcript.js

@@ -8,6 +8,7 @@
  */
 import fs from 'node:fs';
 import path from 'node:path';
+import { computeMetrics } from "./metrics.js";
 /**
  * Default evidence location, resolved against the host repo's cwd. Kept as the
  * default rather than baked in: `reportCases(cases, evidenceDir)` already lets
@@ -17,5 +18,9 @@ export const evidenceDir = path.join(process.cwd(), 'simulations', 'tmp');
 export function writeEvidence(name, transcript, sidecar, dir = evidenceDir) {
     fs.mkdirSync(dir, { recursive: true });
     fs.writeFileSync(path.join(dir, `sim-${name}.json`), JSON.stringify(transcript, null, 2));
-    fs.writeFileSync(path.join(dir, `sim-${name}.run.json`), JSON.stringify(sidecar, null, 2));
+    // Derived here rather than asked of the host: every host would compute the
+    // same thing from the same transcript, and two of these counts are easy to
+    // get wrong (see metrics.ts).
+    const withMetrics = { ...sidecar, metrics: computeMetrics(transcript) };
+    fs.writeFileSync(path.join(dir, `sim-${name}.run.json`), JSON.stringify(withMetrics, null, 2));
 }

package/types.d.ts

@@ -3,6 +3,7 @@
  */
 import type { GatewayExchange } from './gateway-capture.ts';
 import type { Observation } from './page-perception.ts';
+import type { RunMetrics } from './metrics.ts';
 export type SimulationCase = {
     /** Evidence files are named after this; keep it filesystem-safe. */
     name: string;
@@ -32,8 +33,14 @@ export type RunSidecar = {
     valid: boolean;
     error?: string;
     assistantModel: string;
+    /** The tier the background roles ran on (sub-agents, compaction, moderation),
+     *  when the host pins it separately from the assistant's. */
+    toolsModel?: string;
     userModel: string;
     turns: number;
     durationMs: number;
     finishedAt: string;
+    /** Derived from the transcript by `writeEvidence`, so a host gets them without
+     *  asking. Evidence for the judge, never a score. */
+    metrics?: RunMetrics;
 };