@alexkroman1/aai-cli 7.0.0 → 8.1.0

package/dist/templates/support-line/tools/answer_question.ts

@@ -3,6 +3,16 @@ import { z } from "zod";
 import { runCorrectiveRag } from "../procedure.ts";
 import { recordQuestion, supportSlot } from "../shared.ts";
 
+/**
+ * How long the whole corrective loop may run before it gives up.
+ *
+ * Under the runtime's own per-tool deadline (`TOOL_EXECUTION_TIMEOUT_MS`, 30s)
+ * with room for this tool's own bookkeeping, and written as a literal because
+ * that constant is on `@alexkroman1/aai/internal` — a subpath a template may
+ * not import. If the runtime's default ever moves, this is the number to move.
+ */
+const LOOKUP_BUDGET_MS = 28_000;
+
 /**
  * The whole graph, as one tool.
  *
@@ -14,6 +24,33 @@ import { recordQuestion, supportSlot } from "../shared.ts";
  * The verdicts ride back with the answer rather than being swallowed. A
  * grounded-but-not-useful answer is still worth speaking, and the caller is the
  * one who should hear that it may not be the whole story.
+ *
+ * **The loop is given a BUDGET, because the runtime already has one and it is
+ * not this tool's.** The corrective path is up to eleven sequential model calls
+ * — two attempts of grade/generate/ground/use, plus a regeneration — and the
+ * tool executor abandons any tool at `TOOL_EXECUTION_TIMEOUT_MS` (30s), which
+ * an author cannot raise. Measured against a live gateway, the two-attempt path
+ * really does exceed it: the model was handed
+ * `{"error":"Tool \"answer_question\" timed out after 30000ms"}`, which tells
+ * it nothing it can act on and reads to a caller as the line going dead.
+ * `LOOKUP_BUDGET_MS` lands the overrun on the graph's OWN exit instead — the
+ * same "offer to look again, or log a ticket" the design already has for an
+ * interrupted run — so the slow path degrades into a sentence rather than into
+ * an internal error. It does not make the loop faster; it makes running out
+ * something the caller hears about.
+ *
+ * **Re-measured, and the loop is nowhere near the budget today.** Driven
+ * directly against the live gateway on the default model: the single-attempt
+ * path is 2.0-3.1s over four or five model calls, and the two-attempt
+ * CORRECTIVE path — nine calls, one query rewrite — is 2.8s, with each call
+ * 0.3-1.3s. A schema-constrained call is not the expensive kind either (572ms
+ * against 553ms for the same prompt without one), which is the guess anyone
+ * looking at this would make first. So the budget is a GUARD against a slow
+ * gateway rather than a bound the healthy path pushes against, and a question
+ * that trips a rewrite fits with room to spare. Keep it: the measurement above
+ * happened, and what changed is the service's latency, not the number of round
+ * trips. Do not go cutting round trips out of the graph on the strength of the
+ * old paragraph — measure first, the same way.
  */
 export default tool({
   description:
@@ -28,17 +65,29 @@ export default tool({
     try {
       // `ctx.signal` is what stops the graph on a barge-in: this loop is five
       // to nine model calls, and a caller who interrupts on the second should
-      // not be charged for the rest.
-      trace = await runCorrectiveRag(ctx.generate, args.question, ctx.signal);
+      // not be charged for the rest. The budget rides alongside it — see the
+      // module doc on `LOOKUP_BUDGET_MS` — and `AbortSignal.any` is what
+      // combines the two without any unlink bookkeeping.
+      trace = await runCorrectiveRag(
+        ctx.generate,
+        args.question,
+        AbortSignal.any([ctx.signal, AbortSignal.timeout(LOOKUP_BUDGET_MS)]),
+      );
     } catch (err: unknown) {
       // An INTERRUPTED lookup is not a broken one, and the difference is worth
-      // a sentence: `ctx.signal` aborts on a barge-in AND on this call's own
-      // timeout, and telling the model the knowledge base failed would have it
-      // apologize for an outage that did not happen.
+      // a sentence: telling the model the knowledge base failed would have it
+      // apologize for an outage that did not happen. Which of the two signals
+      // fired decides WHICH sentence — a barge-in means the caller is already
+      // talking, and a spent budget means they are still waiting.
       if (err instanceof ProcedureNotFinishedError) {
         return toolFailure(
-          "That lookup was cut short before it finished. Offer to look again, " +
-            "or to log a ticket with log_ticket.",
+          ctx.signal.aborted
+            ? "That lookup was cut short before it finished. Offer to look again, " +
+                "or to log a ticket with log_ticket."
+            : "That lookup ran out of time before it could be graded, so there is no " +
+                "answer to give. Say the check is taking too long, offer a narrower " +
+                "question, and offer to log a ticket with log_ticket. Do not answer " +
+                "from your own knowledge.",
         );
       }
       // A broken model call IS the tool's to report: the model can tell the

package/dist/templates/transcription-workflow/agent.eval.test.ts

@@ -0,0 +1,234 @@
+// Copyright 2026 the AAI authors. MIT license.
+// An EVAL for a WORKFLOW APP: does the run actually do the work? Run it with
+// `aai eval`.
+//
+// `agent.test.ts` asserts about the declaration, drives the four steps one at a
+// time, and covers the WAV arithmetic as pure functions. This drives the WHOLE
+// BODY — `transcribeFlow` from the top — and what it is here to check is the one
+// thing no per-step spec can see: that a recording is really planned, really
+// fanned out, and really stitched back into one transcript in the right order.
+//
+// `describeWorkflowEval` picks the providers for you and says which it picked:
+//
+//   * with `ASSEMBLYAI_API_KEY` — a LIVE run. The `{ live: true }` case below
+//     downloads a real four-minute news clip, converts it with a real ffmpeg,
+//     and puts four real requests through the sync endpoint. That spends money
+//     and about a minute.
+//   * without one — a SCRIPTED run: the same body, the same plan, the same
+//     stitch, with the endpoint answered in memory.
+//
+// Two of the three cases are SCRIPTED IN BOTH MODES, deliberately. Their claims
+// are about the PLAN and the SEAM — which window each request got, and what the
+// merge does where two segments overlap — and those are facts about arithmetic
+// that a live provider can neither confirm nor deny. The live case is the one
+// that answers "is the transcript really of the recording".
+//
+// WHAT NO EVAL HERE COVERS: durability. Imported through vitest with no bundler
+// in the path, a `"use workflow"` body is an ordinary async function — no
+// journal, no replay, and no per-step retry, so the resume-after-segment-27
+// property this template exists to demonstrate is NOT exercised here, and a
+// rate-limited live run fails where a deployed one would have ridden it out.
+// `aai-cli`'s `dev-workflow.scenario.test.ts` is the tier that really resumes a
+// run.
+import { encodeWav } from "@alexkroman1/aai/step";
+import { installStubTranscribe, installStubUploads } from "@alexkroman1/aai/testing/vitest";
+import { describeWorkflowEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import agentDef, { transcribe } from "./agent.ts";
+import { TRANSCRIPT_STREAM } from "./workflows/stitch.ts";
+import { SEGMENT_SECONDS } from "./workflows/wav.ts";
+
+/** The id every case uploads under. */
+const UPLOAD_ID = "upl_eval";
+
+/** 16 kHz mono 16-bit — the format `normalize.ts` converts everything TO. */
+const MONO_16K = { sampleRate: 16_000, channels: 1, bitsPerSample: 16 } as const;
+/** Bytes one second of that format occupies. */
+const BYTES_PER_SECOND = MONO_16K.sampleRate * MONO_16K.channels * (MONO_16K.bitsPerSample / 8);
+
+/**
+ * Long enough to force a THREE-segment fan-out, which is the shape worth
+ * driving: one segment exercises no plan, two exercise one seam, three exercise
+ * a middle segment that is neither first nor last.
+ */
+const SCRIPTED_SECONDS = 200;
+
+/**
+ * A real, parseable, linear-PCM WAV of silence.
+ *
+ * Silence is fine here because the scripted cases never send it anywhere: what
+ * they assert is which BYTE RANGE each request was handed, and the bytes only
+ * have to be as long as the header says. It also takes the fast path through
+ * `normalizeRecording` — `parseWav` accepts it, so no ffmpeg is involved, which
+ * is what keeps these two cases runnable with no binary on `PATH`.
+ */
+const SCRIPTED_WAV = encodeWav(new Uint8Array(SCRIPTED_SECONDS * BYTES_PER_SECOND), MONO_16K);
+
+/** The public sample recording — four minutes of real speech, on a real CDN. */
+const LIVE_RECORDING = "https://assembly.ai/wildfires.mp3";
+
+/** Publish one in-memory upload store, writable because the converter needs one. */
+function publish(bytes: Uint8Array, name: string, type: string) {
+  // `writable: true` is not optional for this template: `normalizeRecording`
+  // writes the CONVERTED file back as a new upload, and a read-only store would
+  // fail that step by name — which is the store telling the truth, and not what
+  // these cases are about.
+  return installStubUploads({ [UPLOAD_ID]: { bytes, name, type } }, { writable: true });
+}
+
+/**
+ * Does this request body carry a WAV header?
+ *
+ * Scanned as BYTES rather than decoded: the body is a multipart envelope around
+ * megabytes of audio, so `String(bytes)` is a comma-joined number list — which
+ * makes the assertion pass or fail for the wrong reason and prints five
+ * megabytes of it when it fails.
+ */
+function carriesWavHeader(body: Uint8Array | string | undefined): boolean {
+  if (!(body instanceof Uint8Array)) return false;
+  const riff = [0x52, 0x49, 0x46, 0x46];
+  for (let at = 0; at <= body.length - riff.length; at++) {
+    if (riff.every((byte, offset) => body[at + offset] === byte)) return true;
+  }
+  return false;
+}
+
+/** The transcript chunks the run streamed, in the order they landed. */
+function streamed(run: { emitted: readonly { namespace: string; chunk: unknown }[] }) {
+  return run.emitted
+    .filter((one) => one.namespace === TRANSCRIPT_STREAM)
+    .map((one) => one.chunk as { index: number; startMs: number; endMs: number; text: string });
+}
+
+describeWorkflowEval(agentDef, (test) => {
+  test("plans the fan-out from the header and gives each request its own window", async ({
+    app,
+  }) => {
+    // Scripted in both modes: the claim is which WINDOW each of the three
+    // requests was handed, and a live endpoint cannot answer that — it can only
+    // transcribe whatever it is sent. This is the case that catches an
+    // off-by-one in the plan, which otherwise produces audio the decoder
+    // transcribes into confident nonsense.
+    publish(SCRIPTED_WAV, "standup.wav", "audio/wav");
+    const provider = installStubTranscribe({
+      text: ["the first stretch", "the middle stretch", "the last stretch"],
+    });
+
+    const run = await app.run(transcribe, { recording: UPLOAD_ID });
+
+    expect(run.error).toBeUndefined();
+    expect(run.status).toBe("completed");
+    const output = run.output;
+    if (output === undefined) expect.fail("a completed run must carry an output");
+
+    // Three segments over 200 seconds at a 90-second stride, and the durations
+    // are the plan: each segment carries the 2-second OVERLAP that stops a cut
+    // landing mid-word, so they deliberately do not abut.
+    expect(output.segments).toBe(3);
+    expect(output.durationMs).toBe(SCRIPTED_SECONDS * 1000);
+    expect(streamed(run).map((chunk) => [chunk.index, chunk.startMs, chunk.endMs])).toEqual([
+      [0, 0, (SEGMENT_SECONDS + 2) * 1000],
+      [1, SEGMENT_SECONDS * 1000, (2 * SEGMENT_SECONDS + 2) * 1000],
+      [2, 2 * SEGMENT_SECONDS * 1000, SCRIPTED_SECONDS * 1000],
+    ]);
+
+    // One sync request per segment, each carrying a WHOLE WAV — the endpoint
+    // decodes every request independently, so a window with no header put back
+    // on it is bytes it refuses.
+    const sync = provider.calls.filter((call) => call.leg === "sync");
+    expect(sync).toHaveLength(3);
+    expect(sync.map((call) => carriesWavHeader(call.body))).toEqual([true, true, true]);
+    // The first two segments are a full stride plus the overlap; the last is the
+    // remainder, and is therefore much smaller. A plan that ran off the end of
+    // the file would show up here as a last segment the size of the others.
+    const sizes = sync.map((call) => call.body?.length ?? 0);
+    expect(sizes[0]).toBeGreaterThan((SEGMENT_SECONDS + 1) * BYTES_PER_SECOND);
+    expect(sizes[2]).toBeLessThan(sizes[0] ?? 0);
+
+    // Stitched in INDEX order, not completion order.
+    expect(output.transcript).toBe("the first stretch the middle stretch the last stretch");
+    // Reported before the fan-out, which is what makes a long run legible.
+    expect(run.reported).toContain("Split 3:20 of audio into 3 segments.");
+  });
+
+  test("drops the repeated words where two segments overlap", async ({ app }) => {
+    // Scripted in both modes for the same reason: the overlap exists so a cut
+    // never lands mid-word, and what makes that free is the merge dropping ONE
+    // copy of the repeated run. A live provider cannot be asked to produce a
+    // seam on demand, and this is where a regression would be invisible — a
+    // missed seam repeats a few words, a false one DELETES speech.
+    publish(SCRIPTED_WAV, "standup.wav", "audio/wav");
+    installStubTranscribe({
+      text: [
+        "the fire moved north through the valley overnight",
+        "through the valley overnight and reached the ridge by dawn",
+        "by dawn the crews had contained it",
+      ],
+    });
+
+    const run = await app.run(transcribe, { recording: UPLOAD_ID });
+
+    expect(run.error).toBeUndefined();
+    const transcript = run.output?.transcript ?? "";
+    expect(transcript).toBe(
+      "the fire moved north through the valley overnight and reached the ridge by dawn the crews had contained it",
+    );
+    // The repeated runs appear once each, which is the whole claim.
+    expect(transcript.match(/overnight/g)).toHaveLength(1);
+    expect(transcript.match(/by dawn/g)).toHaveLength(1);
+  });
+
+  test(
+    "really transcribes a real recording, end to end",
+    async ({ app }) => {
+      // LIVE ONLY, and it is the case that earns the template its name. Three
+      // separate things have to be real for it to pass: ffmpeg has to convert an
+      // MP3 into something the plan can cut, the plan has to cover the whole
+      // recording, and every segment's request has to come back with the words
+      // that are actually in it.
+      //
+      // It needs `ffmpeg` on `PATH` (or `AAI_FFMPEG_PATH`), which is what the
+      // template's own doc says a developer needs for anything that is not
+      // already a WAV. A deployed guest's image installs it.
+      const response = await fetch(LIVE_RECORDING);
+      expect(response.ok).toBe(true);
+      const mp3 = new Uint8Array(await response.arrayBuffer());
+      publish(mp3, "wildfires.mp3", "audio/mpeg");
+
+      const run = await app.run(transcribe, { recording: UPLOAD_ID });
+
+      expect(run.error).toBeUndefined();
+      expect(run.status).toBe("completed");
+      const output = run.output;
+      if (output === undefined) expect.fail("a completed run must carry an output");
+
+      // ffmpeg ran, and the run said so before spending minutes on it.
+      expect(run.reported.some((line) => line.startsWith("Converting wildfires.mp3"))).toBe(true);
+      // The FILENAME a reader sees is the one they uploaded, not the converted
+      // artifact's — `mergeTranscript` reads the ORIGINAL id for exactly this.
+      expect(output.source).toBe("wildfires.mp3");
+
+      // Four and a half minutes, so the plan really fanned out rather than
+      // sending one request.
+      expect(output.segments).toBeGreaterThanOrEqual(3);
+      expect(output.durationMs).toBeGreaterThan(250_000);
+
+      // Every segment came back with words in it, which is how "the whole
+      // recording was covered" is checked rather than assumed: a plan that ran
+      // past the end would leave a silent tail segment here.
+      const chunks = streamed(run);
+      expect(chunks).toHaveLength(output.segments);
+      for (const chunk of chunks) expect(chunk.text.length).toBeGreaterThan(0);
+
+      // And the transcript is of THIS recording — a news segment about smoke
+      // from Canadian wildfires reaching the US east coast.
+      expect(output.transcript).toMatch(/wildfire/i);
+      expect(output.transcript).toMatch(/canada/i);
+      expect(output.transcript).toMatch(/air quality/i);
+      // ~4.5 minutes of speech is several hundred words; a stitch that kept only
+      // the first segment would land far under this.
+      expect(output.words).toBeGreaterThan(400);
+    },
+    { live: true },
+  );
+});

package/dist/templates/travel-concierge/agent.eval.test.ts

@@ -0,0 +1,323 @@
+/// <reference types="vite/client" />
+
+// An EVAL: does the confirmation gate actually gate? Run it with `aai eval`.
+//
+// `agent.test.ts` drives each tool directly and asserts about the state it
+// wrote. That is the right tier for "does `stageAction` refuse a second
+// staging" — and it cannot answer the question this template exists to
+// demonstrate, which is whether the gate still holds when a MODEL is the one
+// picking tools. So these cases drive a real session and read the mechanism off
+// the event stream: which tool ran, what the gated tool answered, and — the
+// claim that matters — whether the caller's booking moved before they said yes.
+//
+// Two modes, per `describeEval`: with a key, a live model chooses the tools;
+// without one, each case's `stubReply` scripts them. A scripted tool call really
+// executes, so the flow gate really runs either way — what a stub run cannot
+// tell you is whether the model would have chosen that tool.
+//
+// **`system-prompt.md` is applied HERE, not by `agent.ts`.** The build discovers
+// the file, so an eval driving the raw default export would run this agent with
+// the FRAMEWORK DEFAULT prompt — and the discipline that prompt imposes is the
+// entire subject of this file. Measured against the default: the model answers
+// product questions from its own knowledge and skips the tools the prompt exists
+// to route it through, so a case run that way measures nothing it claims to.
+
+import { withSystemPrompt } from "@alexkroman1/aai/manifest";
+import { withDiscoveredTools } from "@alexkroman1/aai/testing";
+import { type EvalSession, type EvalTurn, lastStateIn } from "@alexkroman1/aai-runtime/eval";
+import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import { z } from "zod";
+import authoredAgent from "./agent.ts";
+import systemPrompt from "./system-prompt.md?raw";
+
+/**
+ * The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
+ *
+ * The glob is written HERE rather than reached for from a shared helper for the
+ * reason `agent.test.ts` gives — this file ships, so it may not import anything
+ * outside its own template. An eval that forgot it would run an agent with NO
+ * tools and read as a model that refuses to act.
+ */
+const agentDef = withSystemPrompt(
+  withDiscoveredTools(authoredAgent, import.meta.glob("./tools/*.ts", { eager: true })),
+  systemPrompt,
+);
+
+/**
+ * What the BROWSER is sent, as this eval reads it.
+ *
+ * Parsed rather than cast: `state.updated` carries `unknown`, so a projection
+ * that stopped matching FAILS naming the field, where the cast this replaced
+ * handed the assertions `undefined` and failed a line later on something else.
+ * It names only the fields asserted below, so `tripView` may grow without
+ * touching this.
+ */
+const ProjectedTrip = z.object({
+  assistant: z.string(),
+  ticket: z.object({ flightId: z.string() }).nullable(),
+  bookings: z.array(z.unknown()),
+  pending: z.string().nullable(),
+});
+
+/**
+ * The `syncState` frames, in stream order, UP TO the first `confirm_action`.
+ *
+ * This is what makes "a staging tool mutates nothing" assertable against a live
+ * model: the caller's later yes is allowed to move the ticket, so a claim about
+ * the FINAL state is a claim about the conversation rather than about the gate.
+ * What the gate promises is that every frame before the confirmation shows the
+ * booking untouched — and `tripProjection` rides out on `state.updated` after
+ * every tool call, so the stream carries one per step.
+ */
+function framesBeforeConfirm(session: EvalSession): z.infer<typeof ProjectedTrip>[] {
+  const views: z.infer<typeof ProjectedTrip>[] = [];
+  for (const event of session.events()) {
+    if (event.type === "tool.called" && event.toolName === "confirm_action") break;
+    if (event.type === "state.updated") views.push(ProjectedTrip.parse(event.state));
+  }
+  return views;
+}
+
+/**
+ * The latest frame — what the browser would be rendering now. `lastStateIn` is
+ * the SDK's reader for exactly this; the schema is why it is worth passing one.
+ */
+const tripState = (session: EvalSession) => lastStateIn(session.events(), ProjectedTrip);
+
+/**
+ * Drive a whole call, one caller line at a time, and hand back every turn.
+ *
+ * The cases below assert about the turn a MECHANISM fired in rather than about
+ * turn one, because how many turns a desk takes to get there is the model's
+ * business and it moved when the desk gate landed: the flight desk's brief says
+ * to search before quoting anything, so measured live this concierge now spends
+ * its first turn on `to_flight_assistant` and `search_flights` and reads the
+ * fare back before it stages. A case pinned to turn one is a flake with a
+ * misleading name — the same argument `retail`'s eval carries.
+ */
+async function sayAll(session: EvalSession, lines: readonly string[]): Promise<EvalTurn[]> {
+  const turns: EvalTurn[] = [];
+  for (const line of lines) turns.push(await session.say(line));
+  return turns;
+}
+
+/** Every tool call of the call so far, flattened, in order. */
+const callsIn = (turns: readonly EvalTurn[]) => turns.flatMap((turn) => turn.toolCalls);
+
+/** The turn a named tool STAGED something in — the call that answered with the
+ *  read-back rather than with a gate's refusal. */
+const stagingTurn = (turns: readonly EvalTurn[], tool: string) =>
+  turns.find((turn) =>
+    turn.toolCalls.some(
+      (call) => call.name === tool && /awaitingConfirmation/.test(call.result ?? ""),
+    ),
+  );
+
+describeEval(agentDef, (test) => {
+  test(
+    "a sensitive tool stages the change and moves nothing",
+    async ({ session }) => {
+      // Three lines, each of them the same REQUEST and none of them an answer to
+      // a read-back: what is asserted below is that the turn which staged did
+      // not also apply, so a line the model could read as consent ("correct",
+      // "that's right") would be measuring the caller instead of the desk.
+      const turns = await sayAll(session, [
+        "Move my ticket to flight LX52, the Wednesday one.",
+        "I want the Wednesday LX52 instead of the flight I'm on now.",
+        "Put me on LX52 on Wednesday, please.",
+      ]);
+
+      const staging = stagingTurn(turns, "update_ticket");
+      const attempts = callsIn(turns).filter((call) => call.name === "update_ticket");
+      // Named with the whole call, tools AND text: "expected undefined to be
+      // defined" says nothing about a desk that talked its way through three
+      // turns without staging, which is exactly the failure this case caught
+      // while the flight desk's brief had the read-back before the staging.
+      expect(
+        staging,
+        turns
+          .map(
+            (turn, i) =>
+              `turn ${i + 1}: [${turn.toolCalls.map((c) => c.name).join(", ")}] said: ${turn.text}`,
+          )
+          .join("\n"),
+      ).toBeDefined();
+      const staged = staging?.toolCalls.find((call) => call.name === "update_ticket");
+      // The tool answered with the read-back rather than with a receipt.
+      expect(staged?.result).toMatch(/awaitingConfirmation/);
+      // Any attempt that did NOT stage is the DESK GATE refusing:
+      // `update_ticket` belongs to the flight desk, so a model reaching for it
+      // before `to_flight_assistant` is told so and recovers inside the turn.
+      // That refusal is a legal step and it is asserted rather than tolerated —
+      // its own message is what pointed the model at the way in.
+      for (const attempt of attempts.filter((call) => call !== staged)) {
+        expect(attempt.result).toMatch(/to_flight_assistant|Not available yet/);
+      }
+      // The desk asks; it does not decide. A `confirm_action` AFTER the staging
+      // in the same turn is the agent confirming on its own initiative, which
+      // the system prompt forbids in as many words.
+      //
+      // After, not anywhere in the turn: measured live, a model that hears a
+      // second request as a yes reaches for `confirm_action` BEFORE it has
+      // staged anything, the gate refuses it (the case above is where that
+      // refusal is the subject), and it then stages properly. That is a wasted
+      // step rather than an unasked-for change, and folding the two together
+      // would fail this case for the behaviour the next one proves is safe.
+      // Narrowed first: `indexOf` takes a value, and `staged` is optional — the
+      // rewrite Biome offers for `findIndex` over an identity is UNSAFE for
+      // exactly that reason, and the assertion above is what makes an absent
+      // staging call a failure rather than a slice from 0.
+      const stagedAt = staged === undefined ? -1 : (staging?.toolCalls.indexOf(staged) ?? -1);
+      expect(staging?.toolCalls.slice(stagedAt + 1).map((call) => call.name) ?? []).not.toContain(
+        "confirm_action",
+      );
+
+      const views = framesBeforeConfirm(session);
+      const waiting = views.filter((view) => view.pending !== null);
+      expect(waiting.length, "no frame ever showed a staged change").toBeGreaterThan(0);
+      expect(waiting.at(-1)?.pending).toMatch(/LX52/);
+      // THE claim: through every frame up to the confirmation, the ticket is
+      // still the one the caller phoned in on. A staging tool that mutated
+      // would satisfy every assertion above this one.
+      for (const view of views) expect(view.ticket?.flightId).toBe("LX40");
+    },
+    {
+      stubReply: [
+        { tool: "to_flight_assistant", args: { request: "move my ticket to LX52" } },
+        { tool: "update_ticket", args: { flightId: "LX52" } },
+        "That would move you to LX52 on Wednesday, five ninety. Shall I go ahead?",
+      ],
+    },
+  );
+
+  test(
+    "confirm_action is refused while nothing is waiting",
+    async ({ session, mode }) => {
+      const turn = await session.say("Yes, I confirm — go ahead and do it.");
+
+      const attempts = turn.toolCalls.filter((call) => call.name === "confirm_action");
+      // In stub mode the script FORCES the call, so the gate is really
+      // exercised; a live model that declines to call it at all has honoured
+      // the same rule one level earlier, which is why the count is asserted
+      // only where it is determined.
+      if (mode === "stub") expect(attempts).toHaveLength(1);
+      for (const attempt of attempts) {
+        // The refusal names the position and quotes the state's instruction —
+        // that is what the model recovers from, and a gate that ran the body
+        // and then apologized would not carry it.
+        expect(attempt.result).toMatch(/Not available yet/);
+        expect(attempt.result).toMatch(/browsing/);
+      }
+      expect(tripState(session)?.pending ?? null).toBeNull();
+      expect(tripState(session)?.bookings ?? []).toHaveLength(0);
+    },
+    { stubReply: [{ tool: "confirm_action" }, "Nothing is waiting for your yes just now."] },
+  );
+
+  test(
+    "the caller's yes is the only thing that moves the ticket",
+    async ({ session }) => {
+      // Four lines, three of them a yes: which turn the desk stages in and which
+      // it applies in is its own business — the flight desk's brief has it
+      // search first — and saying yes repeatedly is what makes "once each"
+      // below a claim about the MECHANISM rather than about the model's pacing.
+      await sayAll(session, [
+        "Move my ticket to flight LX52, the Wednesday one.",
+        "Correct — LX52 on Wednesday. Please move my ticket to it.",
+        "Yes, that's right — go ahead and change it.",
+        "Yes. Confirm it, please.",
+      ]);
+
+      // The two gated tools' calls, in order.
+      const gated = session
+        .toolCalls()
+        .filter((call) => ["update_ticket", "confirm_action"].includes(call.name));
+      // "Did something" is read POSITIVELY — a staging answers
+      // `awaitingConfirmation`, an apply answers `applied` — so the complement
+      // is a claim rather than a definition.
+      const effective = gated.filter((call) =>
+        /awaitingConfirmation|"applied"/.test(call.result ?? ""),
+      );
+      // Staged first, applied second, once each. Reversed — or a confirm with no
+      // stage — is the regression this template's whole shape exists to prevent.
+      expect(effective.map((call) => call.name)).toEqual(["update_ticket", "confirm_action"]);
+      // Everything else has to be a GATE refusing, and nothing else: the desk
+      // gate turns away an `update_ticket` issued before
+      // `to_flight_assistant`, and the confirmation gate turns away a
+      // `confirm_action` issued before anything is staged (the case above is
+      // where that is the subject). Both are steps a live model really takes,
+      // neither moved the ticket, and neither is what this case is about.
+      for (const call of gated.filter((c) => !effective.includes(c))) {
+        expect(call.result, `${call.name} neither acted nor was refused`).toMatch(
+          /Not available yet|belongs to the/,
+        );
+      }
+
+      const applied = effective.at(-1);
+      expect(applied?.result).toMatch(/LX52/);
+
+      const view = tripState(session);
+      expect(view?.ticket?.flightId).toBe("LX52");
+      expect(view?.pending ?? null).toBeNull();
+    },
+    {
+      stubReply: [
+        // The delegation is in the script because the DESK GATE is real: the
+        // flight desk's tools refuse until `to_flight_assistant` has run, so a
+        // script that skipped it would exercise that refusal rather than this
+        // case's subject.
+        { tool: "to_flight_assistant", args: { request: "move my ticket to LX52" } },
+        { tool: "update_ticket", args: { flightId: "LX52" } },
+        "That's LX52 on Wednesday at five ninety. Confirm it?",
+        { tool: "confirm_action" },
+        "Done — you're on LX52 on Wednesday.",
+      ],
+    },
+  );
+
+  test(
+    "a hotel request goes to the hotel desk rather than being answered at the front",
+    async ({ session }) => {
+      const turn = await session.say(
+        "Leave the flight for now — I need a hotel in Boston for three nights, near the water.",
+      );
+
+      // The INDEX as well as the call, because what came before it is asserted
+      // below and `indexOf` on a possibly-undefined find is worse than both.
+      const handoffAt = turn.toolCalls.findIndex((call) => call.name === "to_hotel_assistant");
+      const handoff = turn.toolCalls[handoffAt];
+      expect(
+        handoff,
+        `tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
+      ).toBeDefined();
+      // The brief IS the tool result, which is the whole port of their
+      // per-assistant prompt onto a session whose prompt is fixed at connect.
+      expect(handoff?.result).toMatch(/hotel desk/);
+      expect(handoff?.args.request).toBeTruthy();
+      // And the stack moved, which is what the caller's sidebar renders and what
+      // `complete_or_escalate` will pop.
+      expect(tripState(session)?.assistant).toBe("hotel");
+      // Anything the model reached for BEFORE delegating was refused, and so
+      // told nothing about Boston hotels: the desk gate is what makes the
+      // handoff above unavoidable rather than requested. This is the assertion
+      // that would have caught the old behaviour even if the model had happened
+      // to delegate afterwards.
+      const before = turn.toolCalls.slice(0, handoffAt);
+      for (const call of before.filter((c) => c.name.startsWith("search_"))) {
+        expect(call.result).toMatch(/belongs to the/);
+      }
+    },
+    // Live only: which desk the model hands the call to is exactly the judgement
+    // a script would be making on its behalf.
+    //
+    // This case is what turned the narrowing into a MECHANISM. It was written
+    // when the delegation was asked for in the prompt and nothing enforced it —
+    // one session has one tool list — and it measured the asking losing 0 of 5:
+    // the model called `search_hotels` from the concierge desk every run. Every
+    // desk tool now checks the stack (`requireDesk` in `shared.ts`), so what
+    // this measures is that the refusal really does route the model through the
+    // desk, inside one turn.
+    { live: true },
+  );
+});