@ggui-ai/negotiator 0.2.0-alpha.1 → 0.2.0-alpha.4

package/src/synth-bench/corpus.ts

@@ -20,11 +20,33 @@
  */
 
 import type { GadgetDescriptor } from '@ggui-ai/protocol';
+import type { RoundTripExpectation } from './round-trip-score.js';
 
 export interface BenchEntry {
   readonly id: string;
   readonly intent: string;
   readonly expected: BenchExpectation;
+  /**
+   * Repair-path seed. When set, the entry exercises the FORGIVING
+   * handshake flow rather than synthesize-from-intent: the runner feeds
+   * this (possibly malformed / suboptimal) agent draft to
+   * `ensureConformingContract`, which lints it → returns verbatim if
+   * clean (origin `agent`) or repairs-in-place if not (origin `synth`).
+   * Mirrors what a real agent submits on `ggui_handshake.blueprintDraft`.
+   * Typed `unknown` because the draft is untrusted — it may not be a
+   * valid `DataContract` (that is the whole point of repairing it).
+   *
+   * Entries WITHOUT a draft run the classic synthesize-from-intent path.
+   */
+  readonly draft?: unknown;
+  /**
+   * Round-trip quality expectation — what the agent intends to do with
+   * the AGREED contract next turn (seed props on `ggui_render`, consume
+   * gestures). The round-trip scorer asserts the produced contract
+   * actually supports it. Only meaningful alongside a `draft` (the
+   * repair corpus); leave absent for shape-only entries.
+   */
+  readonly roundTrip?: RoundTripExpectation;
   /**
    * Per-entry registered gadget catalog. When set, the runner
    * forwards this to `synthesizeContract`'s `appGadgets` option so
@@ -416,13 +438,20 @@ export const BENCH_CORPUS: readonly BenchEntry[] = [
     intent: 'an agent-backed todo list that persists across sessions',
     expected: {
       hasActionSpec: true,
-      hasContextSpec: true,
+      hasContextSpec: false,
       hasStreamSpec: false,
-      hasProps: false,
-      actionNames: ['addTodo', 'deleteTodo', 'add', 'delete', 'create', 'remove'],
-      contextSlots: ['todos', 'items'],
+      hasProps: true,
+      actionNames: [
+        'addTodo',
+        'deleteTodo',
+        'toggle',
+        'add',
+        'delete',
+        'create',
+        'remove',
+      ],
       notes:
-        'agent-backed = explicit persistence = each add/delete IS a discrete event the agent must witness.',
+        'agent-backed/persisted = the agent OWNS the items → todos seed on propsSpec (refreshed via ggui_update); add/delete/toggle are discrete events on actionSpec. contextSpec has no agent-push channel, so an agent-owned persisted list there cannot round-trip — this is the round-trip-correct shape, aligned with list-message-thread / list-file-browser (both props-bearing agent-supplied collections).',
     },
   },
   {
@@ -1124,3 +1153,304 @@ export const BENCH_CORPUS: readonly BenchEntry[] = [
     appGadgets: [CHART_DESCRIPTOR],
   },
 ];
+
+/**
+ * Repair-path corpus — the round-trip QUALITY bench.
+ *
+ * Where {@link BENCH_CORPUS} asks "does synth emit the right SHAPE from
+ * an intent?", this corpus asks the contract-quality question: "when an
+ * agent submits a draft to `ggui_handshake`, does the negotiator's
+ * repair produce a contract that is round-trip-USABLE?" Each entry
+ * carries a real-shaped agent `draft` (run through the production
+ * `ensureConformingContract` create-path) plus a `roundTrip`
+ * expectation describing what the agent does next turn (seed props,
+ * consume gestures). The new round-trip scorer mirrors the render +
+ * consume wire gates, so a "valid-but-broken" repair fails here even
+ * though `lintContract` and the shape scorer pass it.
+ *
+ * The four entries map a 2×2 of (repair-fired?) × (round-trip-survives?):
+ *   - `repair-todo-seed`     — repair FIRES and BREAKS round-trip (RED today;
+ *                              the contract-quality falsifier — synth reshapes
+ *                              the seedable `todos` collection propsSpec →
+ *                              contextSpec, deleting the only agent seed channel).
+ *   - `repair-todo-clean`    — no repair needed (origin `agent`, fast-path);
+ *                              round-trip survives. Pins the no-false-positive case.
+ *   - `repair-form-nextstep` — repair FIRES (dangling nextStep) and PRESERVES the
+ *                              action round-trip. A "good repair" of a non-seed error.
+ *   - `repair-weather-seed`  — repair FIRES (stray wrapper key) and PRESERVES the
+ *                              props round-trip. A "good repair" that keeps propsSpec.
+ *
+ * Used by run-repair-bench(-cli).ts (live LLM probe) and
+ * round-trip-score.test.ts (deterministic scorer pinning).
+ */
+export const REPAIR_CORPUS: readonly BenchEntry[] = [
+  {
+    // THE falsifier. The agent reaches for a JSON-Schema reflex — a
+    // collection on propsSpec plus a wrapper-level `required` names-array
+    // (illegal: required-ness is a per-prop boolean; the wrapper only
+    // takes `description` + `properties`). That single
+    // CTR_SHAPE_UNRECOGNIZED_KEYS bounces the draft off the fast path
+    // into repair. The MINIMAL correct repair is "drop the stray
+    // `required` key — `todos` is render-time seed data, it stays on
+    // propsSpec." But synth's "a mutable collection is ALWAYS contextSpec"
+    // heuristic instead reshapes propsSpec → contextSpec, producing a
+    // VALID contract with no propsSpec — which the accept-path then drops
+    // the agent's `props: { todos }` against. RED on BOTH the shape scorer
+    // (has-props-mismatch) and the round-trip scorer (props-no-home) until
+    // the negotiator learns seed data stays on propsSpec.
+    id: 'repair-todo-seed',
+    intent:
+      'show my todos, let me check off completed ones and add new items',
+    draft: {
+      propsSpec: {
+        description: "The user's todo items",
+        // ⚠️ FATAL: `required` is not a legal key on the propsSpec wrapper.
+        required: ['todos'],
+        properties: {
+          todos: {
+            required: true,
+            schema: {
+              type: 'array',
+              items: {
+                type: 'object',
+                properties: {
+                  id: { type: 'string' },
+                  text: { type: 'string' },
+                  completed: { type: 'boolean' },
+                },
+                required: ['id', 'text', 'completed'],
+              },
+            },
+          },
+        },
+      },
+      actionSpec: {
+        toggleTodo: {
+          label: 'Toggle todo',
+          schema: {
+            type: 'object',
+            properties: { id: { type: 'string' } },
+            required: ['id'],
+          },
+          nextStep: 'todo_toggle',
+        },
+        addTodo: {
+          label: 'Add todo',
+          schema: {
+            type: 'object',
+            properties: { text: { type: 'string' } },
+            required: ['text'],
+          },
+          nextStep: 'todo_add',
+        },
+      },
+      agentCapabilities: {
+        tools: {
+          todo_toggle: {
+            description: 'Flip a todo done/undone',
+            inputSchema: {
+              type: 'object',
+              properties: { id: { type: 'string' } },
+              required: ['id'],
+            },
+          },
+          todo_add: {
+            description: 'Append a new todo',
+            inputSchema: {
+              type: 'object',
+              properties: { text: { type: 'string' } },
+              required: ['text'],
+            },
+          },
+        },
+      },
+    },
+    expected: {
+      // The GOOD outcome: todos stays seedable on propsSpec, toggle/add
+      // stay as actions, NO contextSpec reshape.
+      hasActionSpec: true,
+      hasContextSpec: false,
+      hasStreamSpec: false,
+      hasProps: true,
+      actionNames: ['toggle', 'add'],
+      notes:
+        'contract-quality falsifier: synth currently reshapes the seedable todos collection propsSpec → contextSpec, breaking the agent seed round-trip. Expected GREEN once the negotiator preserves render-time seed data on propsSpec.',
+    },
+    roundTrip: {
+      renderProps: {
+        todos: [{ id: 't1', text: 'Buy milk', completed: false }],
+      },
+      consumableActions: ['toggleTodo', 'addTodo'],
+    },
+  },
+  {
+    // The fast-path / no-false-positive case. Same intent, but the agent
+    // submitted a CLEAN, round-trip-correct draft (todos on propsSpec, no
+    // stray key, resolvable nextStep cross-refs). ensureConformingContract
+    // returns it VERBATIM (origin `agent`, no LLM call). The round-trip
+    // scorer MUST pass — if it flags this, the scorer is over-eager.
+    id: 'repair-todo-clean',
+    intent:
+      'show my todos, let me check off completed ones and add new items',
+    draft: {
+      propsSpec: {
+        description: "The user's todo items",
+        properties: {
+          todos: {
+            required: true,
+            schema: {
+              type: 'array',
+              items: {
+                type: 'object',
+                properties: {
+                  id: { type: 'string' },
+                  text: { type: 'string' },
+                  completed: { type: 'boolean' },
+                },
+                required: ['id', 'text', 'completed'],
+              },
+            },
+          },
+        },
+      },
+      actionSpec: {
+        toggleTodo: {
+          label: 'Toggle todo',
+          schema: {
+            type: 'object',
+            properties: { id: { type: 'string' } },
+            required: ['id'],
+          },
+          nextStep: 'todo_toggle',
+        },
+        addTodo: {
+          label: 'Add todo',
+          schema: {
+            type: 'object',
+            properties: { text: { type: 'string' } },
+            required: ['text'],
+          },
+          nextStep: 'todo_add',
+        },
+      },
+      agentCapabilities: {
+        tools: {
+          todo_toggle: {
+            description: 'Flip a todo done/undone',
+            inputSchema: {
+              type: 'object',
+              properties: { id: { type: 'string' } },
+              required: ['id'],
+            },
+          },
+          todo_add: {
+            description: 'Append a new todo',
+            inputSchema: {
+              type: 'object',
+              properties: { text: { type: 'string' } },
+              required: ['text'],
+            },
+          },
+        },
+      },
+    },
+    expected: {
+      hasActionSpec: true,
+      hasContextSpec: false,
+      hasStreamSpec: false,
+      hasProps: true,
+      actionNames: ['toggle', 'add'],
+      notes:
+        'clean draft — must pass the fast path verbatim (origin agent) and the round-trip scorer. Guards against the scorer false-positiving on a good contract.',
+    },
+    roundTrip: {
+      renderProps: {
+        todos: [{ id: 't1', text: 'Buy milk', completed: false }],
+      },
+      consumableActions: ['toggleTodo', 'addTodo'],
+    },
+  },
+  {
+    // Good repair of a NON-seed error. A contact form: fields are
+    // client-originated draft state (contextSpec), submit is an action.
+    // The draft's submit.nextStep points at a tool that isn't declared
+    // (CTR_REF_NEXT_STEP dangling) → repair fires. The fix (add the tool
+    // or drop the nextStep) has no reason to touch the `submit` action, so
+    // the gesture round-trip survives. No seed props.
+    id: 'repair-form-nextstep',
+    intent: 'a contact form with name, email and message that sends to my backend',
+    draft: {
+      contextSpec: {
+        name: { schema: { type: 'string' }, default: '' },
+        email: { schema: { type: 'string' }, default: '' },
+        message: { schema: { type: 'string' }, default: '' },
+      },
+      actionSpec: {
+        submit: {
+          label: 'Send message',
+          schema: {
+            type: 'object',
+            properties: {
+              name: { type: 'string' },
+              email: { type: 'string' },
+              message: { type: 'string' },
+            },
+            required: ['name', 'email', 'message'],
+          },
+          // ⚠️ dangling: no `send_message` tool declared below.
+          nextStep: 'send_message',
+        },
+      },
+    },
+    expected: {
+      hasActionSpec: true,
+      hasContextSpec: true,
+      hasStreamSpec: false,
+      hasProps: false,
+      actionNames: ['submit', 'send'],
+      contextSlots: ['name', 'email', 'message'],
+      notes:
+        'good repair of a dangling nextStep — the submit gesture must survive so the form round-trips.',
+    },
+    roundTrip: {
+      consumableActions: ['submit'],
+    },
+  },
+  {
+    // Good repair that must KEEP propsSpec. A weather card is the
+    // canonical props-only display case; the agent seeds city/temp/
+    // conditions. The draft carries a stray `additionalProperties` key on
+    // the propsSpec wrapper (illegal — only description+properties allowed)
+    // → CTR_SHAPE_UNRECOGNIZED_KEYS → repair fires. The fix drops the stray
+    // key; propsSpec (and the seed round-trip) must survive.
+    id: 'repair-weather-seed',
+    intent: 'show a weather card for a city with temperature and conditions',
+    draft: {
+      propsSpec: {
+        description: 'Current weather',
+        // ⚠️ FATAL: not a legal key on the propsSpec wrapper.
+        additionalProperties: false,
+        properties: {
+          city: { schema: { type: 'string' }, required: true },
+          temp: { schema: { type: 'number' }, required: true },
+          conditions: { schema: { type: 'string' }, required: true },
+        },
+      },
+    },
+    expected: {
+      hasActionSpec: false,
+      hasContextSpec: false,
+      hasStreamSpec: false,
+      hasProps: true,
+      notes:
+        'good repair of a stray wrapper key — propsSpec must survive so the seed round-trips.',
+    },
+    roundTrip: {
+      renderProps: {
+        city: 'San Francisco',
+        temp: 62,
+        conditions: 'Foggy',
+      },
+    },
+  },
+];

package/src/synth-bench/round-trip-score.ts

@@ -0,0 +1,169 @@
+/**
+ * Round-trip quality scorer.
+ *
+ * The shape scorer ({@link scoreSynthesizedContract}) answers "did the
+ * negotiator emit the right SPECS?" — a validity floor. This scorer
+ * answers the harder question the contract-quality frontier cares
+ * about: "is the produced contract actually USABLE end-to-end?" — i.e.
+ * after handshake agrees on it, can the agent `ggui_render` its seed
+ * data, and `ggui_consume` its gestures, without the wire rejecting or
+ * silently dropping anything?
+ *
+ * Why a structural scorer can't see this: a contract that reshapes a
+ * seedable collection from `propsSpec` to `contextSpec` is STRUCTURALLY
+ * VALID (passes `lintContract`) and PASSES the shape scorer (it has a
+ * contextSpec). Yet it is round-trip-BROKEN: `propsSpec` is the only
+ * agent→client seed channel (and the only `ggui_update` target);
+ * `contextSpec` is client→agent observed state with NO runtime seed
+ * path (`contextSpec.default` is `useState` scaffold, not wire data).
+ * So the agent's `ggui_render({props:{todos:[…]}})` hits a contract
+ * with no `propsSpec` — the accept-path silently drops the props and
+ * the UI renders empty; the override-path hard-throws.
+ *
+ * This scorer mirrors that exact render-handler gate deterministically
+ * (no LLM, no iframe), so the harness catches the "valid-but-broken"
+ * class the shape scorer is blind to. The gate it mirrors lives at
+ * `mcp-server-handlers/src/renders/render.ts:1218-1269` (props-vs-
+ * propsSpec) and the consume action-declaration requirement at
+ * `consume.ts` / `submit-action.ts` (every gesture intent MUST be a
+ * declared `actionSpec` key).
+ */
+
+import type { DataContract } from '@ggui-ai/protocol';
+import { validatePropsData } from '@ggui-ai/protocol';
+
+/**
+ * What the agent intends to do with the agreed contract on the NEXT
+ * turn — the round-trip the produced contract must support. Authored
+ * per repair-corpus entry from the agent's original draft + intent.
+ */
+export interface RoundTripExpectation {
+  /**
+   * The props the agent intends to pass on `ggui_render` once the
+   * contract is agreed — its render-time SEED data. Every key here MUST
+   * land in `contract.propsSpec.properties[key]` (the ONLY agent→client
+   * seed channel — `contextSpec` has no runtime seed path), and the
+   * value MUST satisfy that prop's schema. Mirrors the render handler's
+   * props-vs-propsSpec gate. Omit / leave empty for contracts with no
+   * agent-seeded data (pure forms, broadcasts, counters).
+   */
+  readonly renderProps?: Record<string, unknown>;
+  /**
+   * Gesture intents the agent expects to consume after render (e.g.
+   * `toggleTodo`, `submit`). Each MUST be a declared `actionSpec[*]`
+   * key, else the gesture is structurally unconsumable — the iframe has
+   * no declared intent to dispatch and `ggui_consume` never wakes.
+   */
+  readonly consumableActions?: readonly string[];
+}
+
+export type RoundTripFailureKind =
+  /** Repair bailed to an empty `{}` contract — nothing round-trips. */
+  | 'contract-empty'
+  /**
+   * The agent would seed props but the produced contract declares NO
+   * `propsSpec` — the accept-path drops them / the override-path throws.
+   * The canonical reshape regression (propsSpec → contextSpec).
+   */
+  | 'props-no-home'
+  /** A seed-prop key has no matching `propsSpec.properties` entry. */
+  | 'props-key-unhomed'
+  /** Seed props fail `validatePropsData` against the propsSpec (the
+   *  real wire validator — wrong type, missing required, etc.). */
+  | 'props-rejected'
+  /** A consumable gesture intent is absent from `actionSpec`. */
+  | 'action-undeclared';
+
+export interface RoundTripFailure {
+  readonly kind: RoundTripFailureKind;
+  readonly hint: string;
+}
+
+export interface RoundTripScore {
+  readonly pass: boolean;
+  readonly failures: readonly RoundTripFailure[];
+}
+
+/**
+ * True when a contract declares none of the six spec surfaces — the
+ * `EMPTY_CONTRACT` (`{}`) that `ensureConformingContract` returns when a
+ * draft is unrepairable. Such a contract is structurally valid but
+ * carries no wire at all.
+ */
+function isEmptyContract(contract: DataContract): boolean {
+  return (
+    contract.propsSpec === undefined &&
+    contract.actionSpec === undefined &&
+    contract.streamSpec === undefined &&
+    contract.contextSpec === undefined &&
+    contract.agentCapabilities === undefined &&
+    contract.clientCapabilities === undefined
+  );
+}
+
+/**
+ * Score whether a produced contract supports the agent's intended
+ * round-trip. Pure / deterministic — mirrors the render + consume wire
+ * gates so a "valid-but-round-trip-broken" contract fails here even
+ * though `lintContract` and {@link scoreSynthesizedContract} pass it.
+ */
+export function scoreContractRoundTrip(
+  contract: DataContract,
+  rt: RoundTripExpectation,
+): RoundTripScore {
+  const failures: RoundTripFailure[] = [];
+  const renderProps = rt.renderProps ?? {};
+  const seedKeys = Object.keys(renderProps);
+  const consumable = rt.consumableActions ?? [];
+
+  // An empty contract round-trips nothing it was asked to carry.
+  if (isEmptyContract(contract) && (seedKeys.length > 0 || consumable.length > 0)) {
+    failures.push({
+      kind: 'contract-empty',
+      hint: `repair produced an empty contract ({}); the agent's seed data {${seedKeys.join(', ')}} and gestures [${consumable.join(', ')}] have no wire — the draft was unrepairable.`,
+    });
+  }
+
+  // Props acceptance — mirrors render.ts:1218-1269 (the props-vs-propsSpec gate).
+  if (seedKeys.length > 0) {
+    const propsSpec = contract.propsSpec;
+    if (propsSpec === undefined) {
+      failures.push({
+        kind: 'props-no-home',
+        hint: `agent would seed props {${seedKeys.join(', ')}} on ggui_render, but the produced contract declares NO propsSpec. The accept-path silently drops them (render.ts:1233-1253) and the override-path throws (render.ts:1254-1268); the UI renders empty. A seedable collection was likely reshaped to contextSpec, which has no agent→client seed channel.`,
+      });
+    } else {
+      const properties = propsSpec.properties ?? {};
+      const unhomed = seedKeys.filter((k) => properties[k] === undefined);
+      if (unhomed.length > 0) {
+        failures.push({
+          kind: 'props-key-unhomed',
+          hint: `seed-prop keys {${unhomed.join(', ')}} have no propsSpec.properties entry — ggui_render would reject them as undeclared props.`,
+        });
+      }
+      // The real wire validator the render handler runs (Branch A).
+      const result = validatePropsData(renderProps, propsSpec);
+      if (!result.valid) {
+        failures.push({
+          kind: 'props-rejected',
+          hint: `validatePropsData rejected the agent's seed props against the produced propsSpec: ${result.violations
+            .map((v) => `${v.field}: ${v.message}`)
+            .join('; ')}`,
+        });
+      }
+    }
+  }
+
+  // Consumable gestures — every intent the agent dispatches MUST be a
+  // declared actionSpec key, else the gesture is structurally undefined.
+  for (const action of consumable) {
+    if (contract.actionSpec?.[action] === undefined) {
+      failures.push({
+        kind: 'action-undeclared',
+        hint: `agent expects to consume gesture '${action}', but the produced contract's actionSpec has no '${action}' entry — the iframe has no declared intent to dispatch and ggui_consume never wakes.`,
+      });
+    }
+  }
+
+  return { pass: failures.length === 0, failures };
+}

package/src/synth-bench/run-bench-cli.ts

@@ -18,116 +18,16 @@
  *
  * Bench-only — not exported from the package index.
  */
-import { readFileSync } from 'node:fs';
-import { homedir } from 'node:os';
-import { resolve as pathResolve } from 'node:path';
 import { evaluateAgainstCorpus, formatBenchReport } from './run-bench.js';
-import type { LLMCaller, ToolSchema } from '../llm-caller.js';
 import { contractShape } from './corpus.js';
-
-const ANTHROPIC_API = 'https://api.anthropic.com/v1/messages';
-const DEFAULT_MODEL = 'claude-haiku-4-5';
-
-interface CredsFile {
-  apps?: { global?: { anthropic?: string } };
-}
-
-function resolveAnthropicKey(): string {
-  const envKey = process.env['ANTHROPIC_API_KEY'];
-  if (envKey && envKey.length > 0) return envKey;
-  const credsPath = pathResolve(homedir(), '.ggui', 'credentials.json');
-  let parsed: CredsFile;
-  try {
-    parsed = JSON.parse(readFileSync(credsPath, 'utf8')) as CredsFile;
-  } catch (err) {
-    throw new Error(
-      `bench-synth: could not read ${credsPath} (${err instanceof Error ? err.message : String(err)}). Set ANTHROPIC_API_KEY env var or run \`ggui auth set anthropic\`.`,
-    );
-  }
-  const key = parsed.apps?.global?.anthropic;
-  if (typeof key !== 'string' || key.length === 0) {
-    throw new Error(
-      `bench-synth: no anthropic key found at apps.global.anthropic in ${credsPath}.`,
-    );
-  }
-  return key;
-}
-
-interface AnthropicContentBlock {
-  type: string;
-  name?: string;
-  input?: unknown;
-  text?: string;
-}
-
-interface AnthropicResponse {
-  content?: AnthropicContentBlock[];
-  usage?: { input_tokens?: number; output_tokens?: number };
-  stop_reason?: string;
-  error?: { type?: string; message?: string };
-}
-
-let totalInputTokens = 0;
-let totalOutputTokens = 0;
-
-function buildAnthropicLlmCaller(apiKey: string, model: string): LLMCaller {
-  return {
-    async call(): Promise<string> {
-      throw new Error('bench-synth: text-mode not exercised — synth uses callStructured');
-    },
-    async callStructured<T>(
-      systemPrompt: string,
-      userMessage: string,
-      tool: ToolSchema,
-      maxTokens?: number,
-    ): Promise<T> {
-      // `temperature` deprecated on Haiku 4.5+ — Anthropic rejects with
-      // HTTP 400. `tool_choice: { type: 'tool', name }` below already
-      // binds output to the input_schema; residual stochasticity stays
-      // bounded via canonical-key normalization downstream.
-      const body = {
-        model,
-        max_tokens: maxTokens ?? 1024,
-        system: systemPrompt,
-        messages: [{ role: 'user', content: userMessage }],
-        tools: [
-          {
-            name: tool.name,
-            description: tool.description,
-            input_schema: tool.input_schema,
-          },
-        ],
-        tool_choice: { type: 'tool', name: tool.name },
-      };
-      const res = await fetch(ANTHROPIC_API, {
-        method: 'POST',
-        headers: {
-          'content-type': 'application/json',
-          'x-api-key': apiKey,
-          'anthropic-version': '2023-06-01',
-        },
-        body: JSON.stringify(body),
-      });
-      const json = (await res.json()) as AnthropicResponse;
-      if (!res.ok) {
-        const errType = json.error?.type ?? 'unknown';
-        const errMsg = json.error?.message ?? `HTTP ${res.status}`;
-        throw new Error(`anthropic ${errType}: ${errMsg}`);
-      }
-      if (json.usage) {
-        totalInputTokens += json.usage.input_tokens ?? 0;
-        totalOutputTokens += json.usage.output_tokens ?? 0;
-      }
-      const toolBlock = json.content?.find((b) => b.type === 'tool_use');
-      if (!toolBlock || toolBlock.input === undefined) {
-        throw new Error(
-          `anthropic: no tool_use block in response (stop_reason=${json.stop_reason ?? 'unknown'})`,
-        );
-      }
-      return toolBlock.input as T;
-    },
-  };
-}
+import {
+  DEFAULT_MODEL,
+  HAIKU_4_5_PRICE_INPUT_PER_TOKEN,
+  HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN,
+  buildAnthropicLlmCaller,
+  getTokenUsage,
+  resolveAnthropicKey,
+} from './cli-llm.js';
 
 const SHAPE_FILTERS = [
   'props-only',
@@ -175,12 +75,9 @@ function parseArgs(argv: readonly string[]): CliArgs {
   return result;
 }
 
-const HAIKU_4_5_PRICE_INPUT_PER_TOKEN = 1.0 / 1_000_000;
-const HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN = 5.0 / 1_000_000;
-
 async function main(): Promise<void> {
   const args = parseArgs(process.argv.slice(2));
-  const apiKey = resolveAnthropicKey();
+  const apiKey = resolveAnthropicKey('bench-synth');
   const llm = buildAnthropicLlmCaller(apiKey, args.model);
 
   if (!args.json) {
@@ -216,13 +113,14 @@ async function main(): Promise<void> {
   process.stdout.write(formatBenchReport(report));
   process.stdout.write('\n\n');
 
+  const usage = getTokenUsage();
   const totalCost =
-    totalInputTokens * HAIKU_4_5_PRICE_INPUT_PER_TOKEN +
-    totalOutputTokens * HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN;
+    usage.input * HAIKU_4_5_PRICE_INPUT_PER_TOKEN +
+    usage.output * HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN;
   const callsMade = report.totals.all - report.totals.synthDeclined;
   const costPerCall = callsMade === 0 ? 0 : totalCost / callsMade;
   process.stdout.write(
-    `Tokens: input=${totalInputTokens} output=${totalOutputTokens}\n`,
+    `Tokens: input=${usage.input} output=${usage.output}\n`,
   );
   process.stdout.write(
     `Cost:   total=$${totalCost.toFixed(4)} per-call=$${costPerCall.toFixed(4)}\n`,