@alexkroman1/aai-cli 13.0.0 → 13.2.0

package/dist/templates/personal-finance/agent.eval.test.ts

@@ -16,33 +16,43 @@
 // live rates API.
 
 import agentDef from "virtual:aai/agent";
-import { createVmRunCode, toolResultsIn } from "@alexkroman1/aai-runtime/eval";
+import {
+  createVmRunCode,
+  type EvalTurn,
+  toolArgsIn,
+  toolResultsIn,
+} from "@alexkroman1/aai-runtime/eval";
 import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
 import { expect } from "vitest";
+import { z } from "zod";
 
-type Turn = { toolCalls: readonly { name: string; args: Record<string, unknown> }[] };
+/**
+ * The arguments Penny's two builtins carry, as the wire has them.
+ *
+ * Schemas rather than `String(args.code ?? "")`, which is what `toolArgsIn`
+ * takes one for: `args` is `Record<string, unknown>` — the model wrote it and
+ * nothing validated it — so an argument Penny renamed, or never sent, used to
+ * read as `""`, and the claims below about the code she submitted and the URL
+ * she asked for would have been claims about an empty string. An argument that
+ * stops arriving FAILS here, naming the field.
+ */
+const RunCodeArgs = z.object({ code: z.string() });
+const FetchJsonArgs = z.object({ url: z.string() });
 
 /** The code every `run_code` call in this turn carried, joined. */
-const codeIn = (turn: Turn) =>
-  turn.toolCalls
-    .filter((c) => c.name === "run_code")
-    .map((c) => String(c.args.code ?? ""))
+const codeIn = (turn: EvalTurn) =>
+  toolArgsIn(turn.toolCalls, "run_code", RunCodeArgs)
+    .map((args) => args.code)
     .join("\n");
 
 /** Every URL this turn's `fetch_json` calls asked for. */
-const fetchedUrls = (turn: Turn) =>
-  turn.toolCalls.filter((c) => c.name === "fetch_json").map((c) => String(c.args.url ?? ""));
+const fetchedUrls = (turn: EvalTurn) =>
+  toolArgsIn(turn.toolCalls, "fetch_json", FetchJsonArgs).map((args) => args.url);
 
 /**
- * A `run_code` executor, so these cases can assert the ANSWER.
- *
- * The builtin refuses without one — the Modal container is the security
- * boundary, and off-platform there is none — so a case could assert the CALL and
- * the code it carried, and never what the code came back with.
- * `createVmRunCode()` is a `node:vm` context with a capturing `console.log`,
- * which is enough here: what runs is arithmetic, not a program. It is NOT a
- * sandbox and does not pretend to be one; a deployed agent still gets the
- * refusal.
+ * A `run_code` executor, so the arithmetic cases can assert the ANSWER and not
+ * merely the call — `createVmRunCode`'s own doc carries why the builtin refuses
+ * without one. `fetch_json` needs nothing of the sort: it makes a real request.
  */
 const runCode = createVmRunCode();
 

package/dist/templates/pizza-ordering/agent.eval.test.ts

@@ -13,7 +13,7 @@ import type { SessionEvent } from "@alexkroman1/aai/protocol";
 // SCRIPTED model (its `stubReply`), which still boots this agent, still
 // resolves `tools/`, and still executes the tool a script names — so a stub run
 // proves the wiring and proves nothing about what the agent chose.
-import { lastStateIn, toolResultIn } from "@alexkroman1/aai-runtime/eval";
+import { lastStateIn, statesIn, toolNames, toolResultIn } from "@alexkroman1/aai-runtime/eval";
 import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
 import { expect } from "vitest";
 import { z } from "zod";
@@ -44,9 +44,14 @@ const ProjectedOrder = z.object({
  */
 const lastPushedView = (events: readonly SessionEvent[]) => lastStateIn(events, ProjectedOrder);
 
-/** Every cart the session pushed, in stream order. */
-const pushedViews = (events: readonly SessionEvent[]) =>
-  events.flatMap((e) => (e.type === "state.updated" ? [ProjectedOrder.parse(e.state)] : []));
+/**
+ * Every cart the session pushed, in stream order — `statesIn` is `lastStateIn`'s
+ * plural half, and takes the schema for the same reason.
+ *
+ * The SEQUENCE is the stronger claim: not "the cart is not placed now" but "no
+ * frame the customer ever saw showed it placed".
+ */
+const pushedViews = (events: readonly SessionEvent[]) => statesIn(events, ProjectedOrder);
 
 describeEval(agentDef, (test) => {
   test(
@@ -58,7 +63,7 @@ describeEval(agentDef, (test) => {
 
       // One tool, and the right one: quoting a price without adding the pizza,
       // or adding it twice, are both real findings.
-      expect(turn.toolCalls.map((c) => c.name)).toEqual(["add_pizza"]);
+      expect(toolNames(turn.toolCalls)).toEqual(["add_pizza"]);
       const call = turn.toolCalls[0]!;
       const args = call.args as { size: string; crust: string; toppings: string[] };
       expect(args.size).toBe("large");
@@ -106,7 +111,7 @@ describeEval(agentDef, (test) => {
       // turn leaves the model with nothing to address.
       const turn = await session.say("Actually, make that a large.");
 
-      expect(turn.toolCalls.map((c) => c.name)).toEqual(["update_pizza"]);
+      expect(toolNames(turn.toolCalls)).toEqual(["update_pizza"]);
       const call = turn.toolCalls[0]!;
       expect(call.args).toMatchObject({ pizza_id: 1, size: "large" });
 

package/dist/templates/plan-and-execute/agent.eval.test.ts

@@ -24,7 +24,13 @@
 
 /** The def a DEPLOYED agent runs — see `agent.test.ts` on why the glob is here. */
 import agentDef from "virtual:aai/agent";
-import { type EvalSession, lastStateIn } from "@alexkroman1/aai-runtime/eval";
+import {
+  describeToolCalls,
+  describeTurn,
+  type EvalSession,
+  lastStateIn,
+  toolNames,
+} from "@alexkroman1/aai-runtime/eval";
 import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
 import { expect } from "vitest";
 import { z } from "zod";
@@ -57,8 +63,6 @@ const ProjectedPlan = z.object({
  */
 const planState = (session: EvalSession) => lastStateIn(session.events(), ProjectedPlan);
 
-const named = (calls: readonly { name: string }[]): string[] => calls.map((call) => call.name);
-
 describeEval(agentDef, (test) => {
   test(
     "the stage the desk reports is the flow's, not a guess at the plan",
@@ -122,7 +126,10 @@ describeEval(agentDef, (test) => {
         "I want to work out whether it is cheaper to take the train or fly from London to Edinburgh next month.",
       );
       const started = session.toolCalls().find((call) => call.name === "start_plan");
-      expect(started, `tools called: ${named(session.toolCalls()).join(", ")}`).toBeDefined();
+      // The whole SESSION's calls, not one turn's: the plan may be started on
+      // either utterance, and "expected undefined to be defined" says nothing
+      // about a desk that talked instead. `describeToolCalls` is that sentence.
+      expect(started, describeToolCalls(session.toolCalls())).toBeDefined();
       const planned = planState(session);
       // The tool's own result rides in the message: a planner that FAILED (a
       // gateway error, a schema the provider would not honour) writes nothing,
@@ -131,8 +138,8 @@ describeEval(agentDef, (test) => {
       expect(planned?.plan.length ?? 0).toBeGreaterThan(0);
 
       const worked = await session.say("Yes, go ahead and start on it.");
-      const calls = named(worked.toolCalls).filter((name) => name === "work_next_step").length;
-      expect(calls, `tools called: [${named(worked.toolCalls).join(", ")}]`).toBeGreaterThan(0);
+      const calls = toolNames(worked.toolCalls).filter((name) => name === "work_next_step").length;
+      expect(calls, describeTurn(worked)).toBeGreaterThan(0);
 
       const after = planState(session);
       const done = after?.done ?? [];
@@ -180,7 +187,7 @@ describeEval(agentDef, (test) => {
       const revised = turn.toolCalls.find((call) => call.name === "revise_plan");
       expect(
         revised,
-        `tools called: ${named(turn.toolCalls).join(", ")} — the caller changed the objective, ` +
+        `${describeTurn(turn)} — the caller changed the objective, ` +
           "so this is `revise_plan`, not a plan rewritten by hand",
       ).toBeDefined();
       expect(revised?.args.instruction).toBeTruthy();

package/dist/templates/recap-workflow/agent.eval.test.ts

@@ -62,8 +62,15 @@
  * this file SHIPS — see `agent.test.ts`.
  */
 import agentDef from "virtual:aai/agent";
+import { stubGatewayRoute } from "@alexkroman1/aai/testing";
 import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
-import type { EvalToolCall, EvalWorkflows } from "@alexkroman1/aai-runtime/eval";
+import {
+  describeToolCalls,
+  type EvalToolCall,
+  type EvalWorkflows,
+  toolResultIn,
+  toolResultsIn,
+} from "@alexkroman1/aai-runtime/eval";
 import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
 import { expect } from "vitest";
 import { z } from "zod";
@@ -116,17 +123,24 @@ type ScriptedProvider = {
  * rather than answering an empty 200: a step calling something nobody expected
  * is a finding, where an empty body reads as a provider that said nothing.
  *
+ * The model leg is `stubGatewayRoute`'s, first, because it is the one leg whose
+ * shape this file cannot check: the completion envelope is a WIRE shape, so a
+ * field typed one off does not fail — `stepGenerate` reads no content, reports
+ * an empty completion, and the case blames the recap. The reader routes off the
+ * SDK's own completions PATH and answers `undefined` for everything else, which
+ * is what lets it sit in front of the three batch legs below it.
+ *
  * `hold` keeps the FIRST poll pending, and it is the only way to observe a run
  * that is still going: a durable `sleep` is skipped here, so an unheld run
  * burns its whole poll loop in milliseconds.
  */
 function stubProvider(options: { hold?: boolean; ending?: Ending } = {}): ScriptedProvider {
   const gate = Promise.withResolvers<void>();
+  const model = stubGatewayRoute(RECAP_JSON);
   let polls = 0;
   const stub = installStubStepFetch(async (request) => {
-    if (request.url.includes("/chat/completions")) {
-      return { body: { choices: [{ message: { content: RECAP_JSON } }] } };
-    }
+    const recapped = model.route(request);
+    if (recapped) return recapped;
     if (request.method === "POST") return { body: { id: TRANSCRIPT_ID, status: "queued" } };
     // The compensation. A real DELETE removes the transcript from the account,
     // which is what makes "a failed run leaves nothing behind" a claim rather
@@ -158,21 +172,32 @@ const Cancelled = z.object({ cancelled: z.boolean(), note: z.string() });
  *
  * Parsed rather than regexed: a tool result reaches the event stream as a
  * serialized string, and a shape that stopped matching should fail HERE naming
- * the field instead of handing the next assertion `undefined`.
+ * the field instead of handing the next assertion `undefined`. That is
+ * `toolResultsIn`'s job, and the half it does better than the filter-and-map
+ * this was: a call with no result THROWS naming its position, where
+ * `one.result !== undefined` dropped it — so a tool that was called and never
+ * returned left a shorter list and a case that read the calls it did get.
  */
-function recapStarts(calls: readonly EvalToolCall[]): z.infer<typeof RecapStart>[] {
-  return calls
-    .filter((one) => one.name === "request_recap" && one.result !== undefined)
-    .map((one) => RecapStart.parse(JSON.parse(String(one.result))));
+function recapStarts(calls: readonly EvalToolCall[]): readonly z.infer<typeof RecapStart>[] {
+  return toolResultsIn(calls, "request_recap", RecapStart);
 }
 
-/** The run id the first `request_recap` of this turn reported. */
+/**
+ * The run id the FIRST `request_recap` of this turn reported.
+ *
+ * `recapStarts` rather than `toolResultIn`: a turn is allowed more than one call
+ * here — the case below says so in as many words, a desk that asks twice being
+ * the model's business — and the singular reader refuses a second one on
+ * purpose. What is asserted about the extras is that each was REFUSED with the
+ * run this found.
+ */
 function startedRunId(calls: readonly EvalToolCall[]): string {
   const [first] = recapStarts(calls);
   if (first === undefined) {
-    throw new Error(
-      `the desk called no request_recap: ${calls.map((one) => one.name).join(", ") || "(no tools)"}`,
-    );
+    // `describeToolCalls` is the harness's own sentence for this — including
+    // "called no tools", which is the answer a desk that asked a question
+    // instead of acting gives and the one an empty list reads as truncation.
+    throw new Error(`no request_recap in this turn: ${describeToolCalls(calls)}`);
   }
   return first.runId;
 }
@@ -207,14 +232,19 @@ const START_TURN = [
  * Not tidiness: the scripted provider is unpublished when the test that
  * installed it finishes, so a body still mid-flight would make its next request
  * against whatever the next case publishes — or against the real provider, with
- * a real key. A run drained here COMPLETES rather than failing, and the last
- * thing it does on the way is delete its own transcript: the retention gate's
- * window closes with nobody having answered, which is the safe default. See the
- * header, and the case that pins it.
+ * a real key. `close()` says so on stderr (`EvalRunAbandoned`) when a case
+ * forgets, which is a report rather than a fix — the wait is `settleAll`'s and
+ * the RELEASE is this template's, because what holds the run in flight is this
+ * file's own gate and nothing in the harness can open one.
+ *
+ * A run drained here COMPLETES rather than failing, and the last thing it does
+ * on the way is delete its own transcript: the retention gate's window closes
+ * with nobody having answered, which is the safe default. See the header, and
+ * the case that pins it.
  */
 async function drain(workflows: EvalWorkflows | undefined, provider: ScriptedProvider) {
   provider.release();
-  for (const run of await (workflows?.runs() ?? [])) await workflows?.settle(run.runId, recap);
+  await workflows?.settleAll();
 }
 
 describeEval(
@@ -328,8 +358,7 @@ describeEval(
         const runId = startedRunId(started.toolCalls);
         const turn = await session.say("Forget it — cancel that, please.");
 
-        const cancel = turn.toolCalls.find((one) => one.name === "cancel_recap");
-        const answer = Cancelled.parse(JSON.parse(String(cancel?.result)));
+        const answer = toolResultIn(turn.toolCalls, "cancel_recap", Cancelled);
         expect(answer.cancelled).toBe(true);
         // The sentence is a documented promise of this template, not a
         // decoration: cancellation is NOT cooperative here, so the transcript

package/dist/templates/redline/agent.eval.test.ts

@@ -29,6 +29,7 @@
 // journal, no replay, and no per-step retry, so a rate-limited live run FAILS
 // where a deployed one would have ridden it out. The tier that really resumes a
 // run is `aai-cli`'s `dev-workflow.scenario.test.ts`.
+import { stubGatewayRoute } from "@alexkroman1/aai/testing";
 import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
 import { describeWorkflowEval } from "@alexkroman1/aai-runtime/eval/vitest";
 import { expect } from "vitest";
@@ -52,32 +53,36 @@ const critique = (verdict: "ship" | "revise", score = 8): string =>
     notes: verdict === "ship" ? [] : ["Say what happens to the handover", "Name the start date"],
   });
 
-/** One gateway reply, in the envelope `stepGenerate` reads. */
-const reply = (content: string) => ({ body: { choices: [{ message: { content } }] } });
-
 /**
  * Answer the gateway with `contents`, in order, and record what each stage asked.
  *
- * The last reply repeats, matching `stubGateway`'s convention — a loop cannot
- * know how many calls it will make, and a script that ran out mid-loop would
- * fail on the script rather than on the code. `installStubStepFetch` rather than
- * `installStubGateway`: `stepGenerate` goes through the published `stepFetch`
- * slot, and a published slot BEATS a stubbed global, so stubbing the global here
- * would test a path production does not take.
+ * `stubGatewayRoute` owns both halves this file used to hand-write. The
+ * ENVELOPE, because it is a WIRE shape and a field typed one off does not fail —
+ * `stepGenerate` reads no content, reports an empty completion, and the case
+ * blames the loop. And the CURSOR: the last reply repeats, because a loop cannot
+ * know how many calls it will make, so a script that said one thing forever
+ * could only ever drive it into its budget and one that ran out mid-loop would
+ * fail on the script rather than on the code. Both of those are now the SDK's
+ * one copy rather than this file's second.
+ *
+ * `installStubStepFetch` rather than `installStubGateway`: `stepGenerate` goes
+ * through the published `stepFetch` slot, and a published slot BEATS a stubbed
+ * global, so stubbing the global here would test a path production does not
+ * take. Anything that is not a completion request THROWS — every step in this
+ * body is a model call, so a request the route does not recognise is a finding,
+ * and answering it with a reply anyway is how a stage that started dialling
+ * something else would pass.
  */
 function scriptGateway(contents: readonly string[]) {
-  let next = 0;
-  const fetched = installStubStepFetch(() => {
-    const content = contents.at(Math.min(next, contents.length - 1)) ?? "";
-    next += 1;
-    return reply(content);
+  const model = stubGatewayRoute(contents);
+  installStubStepFetch((request) => {
+    const answered = model.route(request);
+    if (answered === undefined) {
+      throw new Error(`unexpected step request in an eval: ${request.method} ${request.url}`);
+    }
+    return answered;
   });
-  return fetched;
-}
-
-/** Every prompt the gateway was sent, in call order. */
-function promptsOf(fetched: ReturnType<typeof scriptGateway>): string[] {
-  return fetched.calls.map((call) => String(call.body ?? ""));
+  return model;
 }
 
 describeWorkflowEval(agentDef, (test) => {
@@ -140,7 +145,7 @@ describeWorkflowEval(agentDef, (test) => {
     // running, and asking it and then accepting whatever it says is not evidence
     // about the budget. What this pins is the loop's arithmetic — the half a
     // live case cannot reach.
-    const fetched = scriptGateway([
+    const model = scriptGateway([
       DRAFT,
       critique("revise", 4),
       `${DRAFT} It starts on the first Monday of the month.`,
@@ -170,12 +175,15 @@ describeWorkflowEval(agentDef, (test) => {
     ).toBe(true);
     // One draft plus a critique-and-revise pair per round. A loop that critiqued
     // twice, or revised the round it shipped, changes this number.
-    expect(fetched.calls).toHaveLength(1 + 2 * MAX_ROUNDS);
+    expect(model.calls).toHaveLength(1 + 2 * MAX_ROUNDS);
 
     // `briefBlock` is what keeps the three stages from drifting apart, and this is
     // the assertion behind that claim: the writer, the critic AND the reviser were
-    // all shown the same brief and the same must-cover point.
-    const prompts = promptsOf(fetched);
+    // all shown the same brief and the same must-cover point. Read off the
+    // recorded `prompt` — the USER message — rather than off the raw request
+    // body, which is the whole serialized request and would let a `model` id or a
+    // `temperature` satisfy one of these `toContain`s.
+    const prompts = model.calls.map((call) => call.prompt);
     expect(prompts).toHaveLength(1 + 2 * MAX_ROUNDS);
     for (const prompt of prompts) {
       expect(prompt).toContain("quokka");

package/dist/templates/research-workflow/agent.eval.test.ts

@@ -29,8 +29,9 @@
 // endpointing, barge-in, whether two sentences merged into one turn.
 
 import agentDef from "virtual:aai/agent";
+import { stubGatewayRoute } from "@alexkroman1/aai/testing";
 import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
-import type { EvalToolCall, EvalWorkflows } from "@alexkroman1/aai-runtime/eval";
+import { type EvalToolCall, type EvalWorkflows, toolResultIn } from "@alexkroman1/aai-runtime/eval";
 import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
 import { expect } from "vitest";
 import { z } from "zod";
@@ -112,7 +113,17 @@ type ScriptedSteps = {
  * on the live path, so a live case still measures the agent. Anything that is
  * not the gateway THROWS rather than answering 200: an unexpected request from
  * a step is a finding, and a silent empty body would be read as a model that
- * said nothing.
+ * said nothing. `stubGatewayRoute` is what decides which is which, and it
+ * decides on the SDK's own completions PATH — so the script cannot come unstuck
+ * from the step by the two agreeing on a typo, and the envelope it answers with
+ * is the SDK's rather than this file's. That last part is the one worth having:
+ * the envelope is a WIRE shape, so a field typed one off does not fail —
+ * `stepGenerate` reads no content, reports an empty completion, and the case
+ * blames the run.
+ *
+ * The CURSOR is the reader's too: the last reply repeats, which is what a stage
+ * that legitimately calls the model twice needs and what stops a script running
+ * out mid-run and failing on itself.
  *
  * `hold` keeps the FIRST answer pending, which is the only way to observe a run
  * that is still going: a durable `sleep` is skipped here, so an unheld run
@@ -120,15 +131,17 @@ type ScriptedSteps = {
  */
 function scriptSteps(options: { hold?: boolean } = {}): ScriptedSteps {
   const gate = Promise.withResolvers<void>();
-  let served = 0;
+  const model = stubGatewayRoute(MODEL_SCRIPT);
   const stub = installStubStepFetch(async (request) => {
-    if (!request.url.includes("/chat/completions")) {
+    const answered = model.route(request);
+    if (answered === undefined) {
       throw new Error(`unexpected step request in an eval: ${request.method} ${request.url}`);
     }
-    const at = served++;
-    if (options.hold === true && at === 0) await gate.promise;
-    const content = MODEL_SCRIPT[Math.min(at, MODEL_SCRIPT.length - 1)];
-    return { body: { choices: [{ message: { content } }] } };
+    // `model.calls` has already recorded this one, so a length of 1 IS the first
+    // answer — and holding after the route rather than before it keeps the reply
+    // this returns the one the script owed that call.
+    if (options.hold === true && model.calls.length === 1) await gate.promise;
+    return answered;
   });
   return { calls: stub.calls, release: () => gate.resolve() };
 }
@@ -141,20 +154,16 @@ const Started = z.object({
 });
 
 /**
- * The run id a `request_research` call reported.
+ * The run id the `request_research` call reported.
  *
- * Parsed rather than regexed: a tool result reaches the event stream as a
- * serialized string, and a shape that stopped matching should fail HERE naming
- * the field instead of handing the next assertion `undefined`.
+ * `toolResultIn` rather than a `find` and a parse: a tool result reaches the
+ * event stream as a serialized string, and a shape that stopped matching should
+ * fail HERE naming the field instead of handing the next assertion `undefined`.
+ * It throws for the two other ways this can go wrong as well, each naming what
+ * was really called — no such call, and a call that never returned.
  */
 function startedRunId(calls: readonly EvalToolCall[]): string {
-  const call = calls.find((one) => one.name === "request_research");
-  if (call?.result === undefined) {
-    throw new Error(
-      `the desk called no request_research: ${calls.map((one) => one.name).join(", ") || "(no tools)"}`,
-    );
-  }
-  return Started.parse(JSON.parse(call.result)).runId;
+  return toolResultIn(calls, "request_research", Started).runId;
 }
 
 /** Every tool call in this turn that READS a run, whichever the model picked. */
@@ -177,12 +186,13 @@ const START_TURN = [
  * Not tidiness: the scripted `stepFetch` is unpublished when the test that
  * installed it finishes, so a body still mid-flight would make its next model
  * call against whatever the next case publishes — or against the real gateway.
+ * `close()` reports that on stderr (`EvalRunAbandoned`) rather than fixing it:
+ * the wait is `settleAll`'s, and the RELEASE stays here, because what holds the
+ * run in flight is this file's own gate and nothing in the harness can open one.
  */
 async function drain(workflows: EvalWorkflows | undefined, steps: ScriptedSteps): Promise<void> {
   steps.release();
-  for (const run of await (workflows?.runs() ?? [])) {
-    await workflows?.settle(run.runId, research);
-  }
+  await workflows?.settleAll();
 }
 
 describeEval(

package/dist/templates/retail/agent.eval.test.ts

@@ -31,7 +31,7 @@ import type { SessionEvent } from "@alexkroman1/aai/protocol";
 // What no eval here can see: anything below the audio boundary. Whether a
 // caller reading an order number in bursts lands as one turn is a property of
 // endpointing, and these fake speech stages remove it.
-import { type EvalSession, type EvalTurn, lastStateIn } from "@alexkroman1/aai-runtime/eval";
+import { describeTurn, lastStateIn, toolNames, turnCalling } from "@alexkroman1/aai-runtime/eval";
 import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
 import { expect } from "vitest";
 import { z } from "zod";
@@ -93,26 +93,6 @@ function statusOf(events: readonly SessionEvent[], orderId: string): string | un
 const refusalAt = (state: string) =>
   new RegExp(`Not available yet: this conversation is at [\\\\"]*${state}`);
 
-/**
- * Drive a whole call, one caller line at a time, and hand back every turn.
- *
- * The cases below assert about the turn a MECHANISM fired in rather than about
- * turn number two, because how many turns a desk takes to get there is the
- * model's business and it really does vary: measured live, this agent reads the
- * order back out of `get_order_details` and asks before it stages, so the
- * staging call has landed in turn two, three and four across runs. A case
- * pinned to a turn index is a flake with a misleading name.
- */
-async function sayAll(session: EvalSession, lines: readonly string[]): Promise<EvalTurn[]> {
-  const turns: EvalTurn[] = [];
-  for (const line of lines) turns.push(await session.say(line));
-  return turns;
-}
-
-/** The turn a named tool was called in, if any. */
-const turnCalling = (turns: readonly EvalTurn[], tool: string) =>
-  turns.find((t) => t.toolCalls.some((c) => c.name === tool));
-
 /** One line the caller says to identify themselves, and the scripted tool call
  *  that answers it — the first turn of three of these four cases. */
 const AUTH_TURN = [
@@ -175,7 +155,7 @@ describeEval(retailAgent, (test) => {
       // turn, and whether it spends one is not something an eval should pin.
       // Every assertion below is about the turn the staging landed in, so a
       // later apply cannot affect any of them.
-      const turns = await sayAll(session, [
+      const turns = await session.sayAll([
         `My email is ${CALLER_EMAIL}.`,
         "I'd like to cancel my pending order — I ordered it by mistake.",
         "Yes, please go ahead and cancel it.",
@@ -183,8 +163,13 @@ describeEval(retailAgent, (test) => {
         "Yes. Cancel it, please.",
       ]);
 
+      // The turn the staging landed in, whichever it was — measured live it has
+      // been turn two, three and four. `turnCalling` throws when no turn staged
+      // at all, naming every turn's tool list: a desk that talked through all
+      // five without staging is the finding, and "expected undefined to be
+      // defined" is not a report of it.
       const staging = turnCalling(turns, "cancel_pending_order");
-      const staged = staging?.toolCalls.find((c) => c.name === "cancel_pending_order");
+      const staged = staging.toolCalls.find((c) => c.name === "cancel_pending_order");
       expect(staged?.result).toMatch(/NOTHING HAS CHANGED YET/);
       // The gate is a POSITION, and this is it moving: the tool reported the
       // state it landed in, which is the only state `confirm_change` is legal
@@ -192,13 +177,13 @@ describeEval(retailAgent, (test) => {
       expect(staged?.result).toMatch(/serving\.awaitingConfirmation/);
       // A change cannot be described and applied in the same turn. This is the
       // property the prose in the system prompt could never have.
-      expect(staging?.toolCalls.map((c) => c.name)).not.toContain("confirm_change");
+      expect(toolNames(staging.toolCalls)).not.toContain("confirm_change");
       // And after the turn that staged it, the store really is untouched — read
       // off the projection the BROWSER was sent in that same turn.
-      expect(statusOf(staging?.events ?? [], PENDING_ORDER)).toBe("pending");
-      expect(projection(staging?.events ?? [])?.pending?.kind).toBe("cancel_pending_order");
+      expect(statusOf(staging.events, PENDING_ORDER)).toBe("pending");
+      expect(projection(staging.events)?.pending?.kind).toBe("cancel_pending_order");
       // Step 2 of the policy: read it back and ask.
-      expect(staging?.text).toMatch(/\?/);
+      expect(staging.text).toMatch(/\?/);
     },
     { stubReply: [...AUTH_TURN, ...STAGE_TURN, "Cancelling it now — one moment."] },
   );
@@ -210,7 +195,7 @@ describeEval(retailAgent, (test) => {
       // in is its own business, and saying yes repeatedly is what makes
       // "exactly once" below a claim about the MECHANISM rather than about the
       // model's pacing.
-      await sayAll(session, [
+      await session.sayAll([
         `My email is ${CALLER_EMAIL}.`,
         "Please cancel my pending order — I ordered it by mistake.",
         "Yes, that's right, go ahead.",
@@ -231,7 +216,7 @@ describeEval(retailAgent, (test) => {
         expect(extra.result).toMatch(/Not available yet/);
       }
       // And it came after the stage, never instead of it.
-      const names = session.toolCalls().map((c) => c.name);
+      const names = toolNames(session.toolCalls());
       expect(names.indexOf("cancel_pending_order")).toBeGreaterThanOrEqual(0);
       expect(names.indexOf("confirm_change")).toBeGreaterThan(
         names.indexOf("cancel_pending_order"),
@@ -262,11 +247,10 @@ describeEval(retailAgent, (test) => {
       // Named first, and with a message: a live model that answers the request
       // with a question instead of the tool leaves `transfer` undefined, and
       // `.toMatch()` on it reports only "expected a string, got undefined" —
-      // which says nothing about what the desk actually did.
-      expect(
-        transfer,
-        `tools called: [${handoff.toolCalls.map((c) => c.name).join(", ")}]; said: ${handoff.text}`,
-      ).toBeDefined();
+      // which says nothing about what the desk actually did. `describeTurn` is
+      // that sentence, done by the harness: the tools it called and what it
+      // said, plus whether the reply was cancelled.
+      expect(transfer, describeTurn(handoff)).toBeDefined();
       // The terminal state is what makes "say nothing else after this" a
       // property of the agent rather than a line in its prompt: `done` is the
       // flow saying there is nowhere left to go.