@alexkroman1/aai-cli 6.11.0 → 8.0.0

package/dist/templates/spoken-summary/agent.ts

@@ -19,7 +19,7 @@
  * The first three are ordinary step work. The fourth was impossible until two
  * things existed, and they are what this template is the reference use of:
  *
- * - **`stepSpeak`** (`@alexkroman1/aai/utils`) synthesizes from inside a step.
+ * - **`stepSpeak`** (`@alexkroman1/aai/step`) synthesizes from inside a step.
  *   The session TTS surface cannot be used here at all: a `TtsSession` is an
  *   event stream wired into a live pipeline's playback, with a turn tracker and
  *   barge-in behind it, and a step has no turn to be part of and has to return
@@ -76,8 +76,9 @@
 
 import { workflow, workflowApp } from "@alexkroman1/aai";
 import { ASSEMBLYAI_TTS_DEFAULT_VOICE, ASSEMBLYAI_TTS_VOICES } from "@alexkroman1/aai/tts";
+import type { WorkflowDef } from "@alexkroman1/aai/workflow-api";
 import { z } from "zod";
-import { spokenSummaryFlow } from "./workflows/summarize.ts";
+import { type SpokenSummary, spokenSummaryFlow } from "./workflows/summarize.ts";
 
 /**
  * The voices the form offers.
@@ -102,28 +103,40 @@ const VOICES = Object.entries(ASSEMBLYAI_TTS_VOICES)
  */
 const [FIRST_VOICE = ASSEMBLYAI_TTS_DEFAULT_VOICE, ...OTHER_VOICES] = VOICES;
 
+/**
+ * The run input, as its own const.
+ *
+ * Named rather than inline because {@link spokenSummary} carries an explicit
+ * type, and that annotation is what lets `workflows/summarize.ts` name
+ * `WorkflowInputOf<typeof spokenSummary>` for its body's parameter: the body's
+ * own signature would otherwise be part of what infers this declaration's type,
+ * and TypeScript refuses the cycle (`TS7022`).
+ */
+const spokenSummaryInput = z.object({
+  // A plain string, because an upload id is what the run really receives.
+  // What makes it a file picker rather than a text box is the `uploads` line
+  // below.
+  recording: z.string().describe("A recording to summarize — WAV, MP3 or M4A"),
+  // An enum, so the form renders a SELECT rather than a text box — which is
+  // the whole reason the list is derived above rather than left free-form.
+  // Optional, so the SDK's own default voice applies when nobody chooses.
+  voice: z
+    .enum([FIRST_VOICE, ...OTHER_VOICES])
+    .optional()
+    .describe("Voice to read the summary in"),
+});
+
 /**
  * The declaration: schema, description, and the directive body.
  *
  * Exported so `WorkflowOutputOf<typeof spokenSummary>` names the output type in
  * one place — including from `client.tsx`, where `import type` is erased and so
- * bundles nothing server-side.
+ * bundles nothing server-side — and so `workflows/summarize.ts` can name
+ * `WorkflowInputOf<typeof spokenSummary>` for the body's parameter.
  */
-export const spokenSummary = workflow({
+export const spokenSummary: WorkflowDef<typeof spokenSummaryInput, SpokenSummary> = workflow({
   description: "Transcribe a recording, summarize it, and read the summary back as audio",
-  input: z.object({
-    // A plain string, because an upload id is what the run really receives.
-    // What makes it a file picker rather than a text box is the `uploads` line
-    // below.
-    recording: z.string().describe("A recording to summarize — WAV, MP3 or M4A"),
-    // An enum, so the form renders a SELECT rather than a text box — which is
-    // the whole reason the list is derived above rather than left free-form.
-    // Optional, so the SDK's own default voice applies when nobody chooses.
-    voice: z
-      .enum([FIRST_VOICE, ...OTHER_VOICES])
-      .optional()
-      .describe("Voice to read the summary in"),
-  }),
+  input: spokenSummaryInput,
   // The one line that makes the form take a file: `<WorkflowFields>` renders a
   // picker for this property, `useWorkflowSubmit` stores the chosen file, and
   // the step that transcribes it reads it back with `readUpload`.

package/dist/templates/spoken-summary/client.tsx

@@ -36,18 +36,19 @@ import "@alexkroman1/aai-ui/styles.css";
 // ERASED at build time, so naming the agent's own type costs the browser bundle
 // nothing — and it is what stops this file restating a shape
 // `workflows/summarize.ts` already declares.
-import type { WorkflowOutputOf } from "@alexkroman1/aai";
+import { formatDuration } from "@alexkroman1/aai/utils";
+import type { WorkflowOutputOf } from "@alexkroman1/aai/workflow-api";
 import {
   createWorkflowApi,
   Form,
   page,
   SubmitButton,
   UploadProgressBar,
+  useDownloadUrl,
   useWorkflowSubmit,
   WorkflowFields,
   WorkflowProgress,
 } from "@alexkroman1/aai-ui";
-import { useEffect, useState } from "react";
 import type { spokenSummary } from "./agent.ts";
 
 /** What a completed run reports, derived from the workflow rather than restated. */
@@ -84,58 +85,16 @@ function captionsUrl(text: string, durationMs: number): string {
   return `data:text/vtt;charset=utf-8,${encodeURIComponent(vtt)}`;
 }
 
-/** Seconds a person can read, from the milliseconds a run reports. */
-function duration(ms: number): string {
-  const total = Math.round(ms / 1000);
-  return `${Math.floor(total / 60)}:${String(total % 60).padStart(2, "0")}`;
-}
-
-/**
- * The finished run's audio, as something the browser will play.
- *
- * A hook rather than four lines in the component because the CLEANUP is the
- * part worth keeping in one place: an object URL pins its blob for the life of
- * the document, so it is revoked when the id changes and when the page goes
- * away. The `cancelled` flag covers the other half — a second run settling
- * while the first download is still in flight would otherwise set state from
- * the stale one.
- */
-function useAudioUrl(uploadId: string | undefined): { url?: string; error?: string } {
-  const [state, setState] = useState<{ url?: string; error?: string }>({});
-
-  useEffect(() => {
-    if (uploadId === undefined) {
-      setState({});
-      return;
-    }
-    let cancelled = false;
-    let objectUrl: string | undefined;
-    api
-      .download(uploadId)
-      .then((blob) => {
-        if (cancelled) return;
-        objectUrl = URL.createObjectURL(blob);
-        setState({ url: objectUrl });
-      })
-      .catch((err: unknown) => {
-        if (!cancelled) setState({ error: err instanceof Error ? err.message : String(err) });
-      });
-    return () => {
-      cancelled = true;
-      if (objectUrl !== undefined) URL.revokeObjectURL(objectUrl);
-    };
-  }, [uploadId]);
-
-  return state;
-}
-
 export function App() {
   // The generic is what makes `run.status === "completed"` narrow to a TYPED
   // `run.output` instead of `unknown`.
   const { submit, run, pending, upload, pauseUpload, resumeUpload, error } =
     useWorkflowSubmit<Summary>(WORKFLOW, { api });
   const output = run?.status === "completed" ? run.output : undefined;
-  const audio = useAudioUrl(output?.audio);
+  // `useDownloadUrl` is the SDK's: the byte route takes the agent's bearer, so the
+  // bytes have to be FETCHED and handed to the element as an object URL — and the
+  // object URL has to be revoked, which is the half a page written by hand forgets.
+  const audio = useDownloadUrl(output?.audio, { api });
 
   return (
     <main className="mx-auto flex max-w-2xl flex-col gap-6 p-8">
@@ -169,7 +128,7 @@ export function App() {
           <div className="flex flex-col gap-1">
             <h2 className="text-xl">{output.headline}</h2>
             <p className="text-sm opacity-70">
-              {output.source} · {duration(output.durationMs)} · {output.words} words
+              {output.source} · {formatDuration(output.durationMs)} · {output.words} words
             </p>
           </div>
 
@@ -181,8 +140,9 @@ export function App() {
 
           <section className="flex flex-col gap-2">
             <h3 className="text-sm font-medium opacity-70">
-              Read aloud · {duration(output.audioDurationMs)}
+              Read aloud · {formatDuration(output.audioDurationMs)}
             </h3>
+            {audio.pending && <p className="text-sm opacity-70">Fetching the audio…</p>}
             {audio.error !== undefined && (
               <p className="text-red-600">Could not load the audio: {audio.error}</p>
             )}

package/dist/templates/spoken-summary/workflows/summarize.ts

@@ -24,7 +24,7 @@
  *   run. The bytes go to the store, the output carries the id, and the page
  *   turns it back into something to play with `api.download(id)`.
  *
- * Both are on `@alexkroman1/aai/utils`, imported from THERE rather than the
+ * Both are on `@alexkroman1/aai/step`, imported from THERE rather than the
  * root: a `workflows/*.ts` module is bundled separately by the WDK builder, so
  * the root barrel's module graph would ride into the step bundle.
  *
@@ -49,19 +49,17 @@
  * expensive one.
  */
 
-import { throwStepError } from "@alexkroman1/aai/step-errors";
-import {
-  omitUndefined,
-  report,
-  stepGenerateJson,
-  stepSpeak,
-  TRANSCRIBE_API,
-  writeUpload,
-} from "@alexkroman1/aai/utils";
+import { report, stepSpeak, TRANSCRIBE_API, writeUpload } from "@alexkroman1/aai/step";
+import { stepGenerateJsonClassified } from "@alexkroman1/aai/step-errors";
+import { countWords, omitUndefined } from "@alexkroman1/aai/utils";
+// ERASED at build time, so the body can name the schema's own output type without
+// a runtime cycle back through `agent.ts` — the same mechanism `client.tsx` uses
+// for `WorkflowOutputOf`.
+import type { WorkflowInputOf } from "@alexkroman1/aai/workflow-api";
 import { sleep } from "workflow";
 import { z } from "zod";
+import type { spokenSummary } from "../agent.ts";
 import {
-  countWords,
   createJob,
   MAX_POLLS,
   POLL_INTERVAL,
@@ -128,13 +126,9 @@ export type SpokenSummary = {
 };
 
 /** Transcribe a recording, summarize it, and read the summary back. */
-export async function spokenSummaryFlow(input: {
-  recording: string;
-  // `| undefined` explicitly, not merely optional: `exactOptionalPropertyTypes`
-  // is on repo-wide, and what a zod `.optional()` infers is a property that may
-  // be PRESENT and undefined.
-  voice?: string | undefined;
-}): Promise<SpokenSummary> {
+export async function spokenSummaryFlow(
+  input: WorkflowInputOf<typeof spokenSummary>,
+): Promise<SpokenSummary> {
   "use workflow";
 
   const transcript = await transcribe(input.recording);
@@ -188,7 +182,7 @@ export async function summarize(
   "use step";
 
   await report("Summarizing the transcript.");
-  const reply = await stepGenerateJson(
+  const reply = await stepGenerateJsonClassified(
     "Summarize this transcript of a recording.\n\n" +
       "Answer with JSON only, in this shape:\n" +
       `{"headline": "...", "points": ["..."], "spoken": "..."}\n\n` +
@@ -204,9 +198,10 @@ export async function summarize(
       system: "You summarize recordings. You answer with JSON and nothing else.",
       schema: SummaryReply,
     },
-    // Classified off the gateway's own status: a 429 is worth another attempt
-    // and a 400 is not, and `throwStepError` is what tells the DevKit which.
-  ).catch(throwStepError);
+    // The `Classified` caller is `stepGenerateJson` plus `throwStepError`, which
+    // reads the gateway's own status: a 429 is worth another attempt and a 400
+    // is not, and that is what tells the DevKit which.
+  );
 
   return { headline: reply.headline, points: reply.points.slice(0, POINTS), spoken: reply.spoken };
 }

package/dist/templates/spoken-summary/workflows/transcribe.ts

@@ -12,7 +12,8 @@
  *
  * **Every one of them is four lines, because the SDK owns the endpoint.**
  * `stepTranscribeUpload` / `stepTranscribeSubmit` / `stepTranscribePoll` on
- * `@alexkroman1/aai/utils` carry the URL, the raw-key auth, the windowed
+ * `@alexkroman1/aai/step` — reached here through their `*Classified` callers on
+ * `@alexkroman1/aai/step-errors` — carry the URL, the raw-key auth, the windowed
  * streaming upload, the PLURAL `speech_models` field and the failure
  * classification — all of which this file used to spell out, and all of which
  * `transcription-workflow` used to spell out again, differently worded and
@@ -37,14 +38,13 @@
  * the one leg that should be as boring as possible.
  */
 
-import { throwStepError } from "@alexkroman1/aai/step-errors";
+import { report, uploadInfo } from "@alexkroman1/aai/step";
 import {
-  report,
-  stepTranscribePoll,
-  stepTranscribeSubmit,
-  stepTranscribeUpload,
-  uploadInfo,
-} from "@alexkroman1/aai/utils";
+  stepTranscribePollClassified,
+  stepTranscribeSubmitClassified,
+  stepTranscribeUploadClassified,
+} from "@alexkroman1/aai/step-errors";
+import { countWords, formatBytes } from "@alexkroman1/aai/utils";
 
 /** How long between polls of a submitted job. */
 export const POLL_INTERVAL = "10s";
@@ -79,16 +79,20 @@ export type Transcript = {
  * expires before the next step runs; that costs one fresh upload, once, instead
  * of five.
  *
- * `.catch(throwStepError)` is what turns the SDK's `TranscribeError` into the
- * DevKit's verdict — a missing key and a 400 stop, a 429 waits as long as the
- * service asked. Every step here ends the same way for the same reason.
+ * The `Classified` callers on `@alexkroman1/aai/step-errors` are the SDK's own
+ * `stepTranscribe*` plus `throwStepError` and nothing else, which is what turns
+ * the SDK's `TranscribeError` into the DevKit's verdict — a missing key and a
+ * 400 stop, a 429 waits as long as the service asked. Every step here ends the
+ * same way for the same reason.
  */
 export async function uploadToProvider(uploadId: string): Promise<{ audioUrl: string }> {
   "use step";
 
   const stored = await uploadInfo(uploadId);
-  await report(`Uploading ${stored.name || uploadId} (${mb(stored.size)}) for transcription.`);
-  return await stepTranscribeUpload(uploadId).catch(throwStepError);
+  await report(
+    `Uploading ${stored.name || uploadId} (${formatBytes(stored.size)}) for transcription.`,
+  );
+  return await stepTranscribeUploadClassified(uploadId);
 }
 
 /** Retries beyond the default 3: an upload is the one call here worth another attempt. */
@@ -98,7 +102,7 @@ uploadToProvider.maxRetries = 5;
 export async function createJob(audioUrl: string): Promise<{ id: string }> {
   "use step";
 
-  const job = await stepTranscribeSubmit(audioUrl).catch(throwStepError);
+  const job = await stepTranscribeSubmitClassified(audioUrl);
   await report(`Transcribing — job ${job.id}.`);
   return job;
 }
@@ -118,7 +122,7 @@ export async function pollTranscript(
 ): Promise<{ done: false } | { done: true; transcript: Transcript }> {
   "use step";
 
-  const progress = await stepTranscribePoll(id).catch(throwStepError);
+  const progress = await stepTranscribePollClassified(id);
   if (!progress.done) return { done: false };
 
   const stored = await uploadInfo(uploadId);
@@ -132,14 +136,3 @@ export async function pollTranscript(
     },
   };
 }
-
-/** Words in a transcript, for the counts a page shows. */
-export function countWords(text: string): number {
-  const trimmed = text.trim();
-  return trimmed.length === 0 ? 0 : trimmed.split(/\s+/).length;
-}
-
-/** A size a person can read, because the number that matters is the scale. */
-function mb(bytes: number): string {
-  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
-}

package/dist/templates/support-line/agent.eval.test.ts

@@ -0,0 +1,273 @@
+/// <reference types="vite/client" />
+
+// An EVAL: does the line really refuse to say what it cannot ground? Run it
+// with `aai eval`.
+//
+// `agent.test.ts` drives the retriever and each CRAG node directly. What it
+// cannot see is the claim this template exists to make — that a MODEL holding
+// this tool list cannot state a fee it did not just retrieve. So these cases
+// drive a real session and read the verdicts off the tool results: which
+// documents the grader passed, whether the answer was grounded, and whether an
+// answer it could not ground was WITHHELD.
+//
+// **One thing a SCRIPTED run cannot reach: `ctx.generate` with a SCHEMA.** The
+// stub model registers as an LLM provider, so `ctx.generate` resolves it, and it
+// answers a plain text generation from the script. Every node in this template's
+// loop passes a `schema` though (`binaryScore`), and that overload comes back
+// `{"error":"The knowledge base lookup failed: No output generated."}` — so the
+// whole corrective loop is unreachable scripted. That is the template's central
+// tool, so the two cases that run without a key are written as invariants both
+// modes satisfy (a failed lookup is REPORTED, never answered around), and the
+// two that read a real verdict are `{ live: true }`.
+//
+// **`system-prompt.md` is applied HERE, not by `agent.ts`.** The build discovers
+// the file, so an eval driving the raw default export would run this agent with
+// the FRAMEWORK DEFAULT prompt — and the discipline that prompt imposes is the
+// entire subject of this file. Measured against the default: the model answers
+// product questions from its own knowledge and skips the tools the prompt exists
+// to route it through, so a case run that way measures nothing it claims to.
+
+import { withSystemPrompt } from "@alexkroman1/aai/manifest";
+import { withDiscoveredTools } from "@alexkroman1/aai/testing";
+import { type EvalSession, toolResultIn } from "@alexkroman1/aai-runtime/eval";
+import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import { z } from "zod";
+import authoredAgent from "./agent.ts";
+import systemPrompt from "./system-prompt.md?raw";
+
+/** The def a DEPLOYED agent runs — see `agent.test.ts` on why the glob is here. */
+const agentDef = withSystemPrompt(
+  withDiscoveredTools(authoredAgent, import.meta.glob("./tools/*.ts", { eager: true })),
+  systemPrompt,
+);
+
+/**
+ * What the BROWSER is sent, as this eval reads it.
+ *
+ * Parsed rather than cast: `state.updated` carries `unknown`, so a projection
+ * that stopped matching FAILS naming the field, where the cast this replaced
+ * handed the assertions `undefined` and failed a line later on something else.
+ * It names only the fields asserted below, so `supportView` may grow without
+ * touching this.
+ */
+const ProjectedSupport = z.object({
+  ticket: z.string().nullable(),
+  trace: z
+    .object({
+      exhausted: z.boolean(),
+      docs: z.array(z.object({ relevant: z.boolean() })),
+      steps: z.array(z.object({ node: z.string() })),
+    })
+    .nullable(),
+});
+
+/**
+ * What `answer_question` answered, validated.
+ *
+ * All-optional, and that is the shape of the CONTRACT rather than laxness: the
+ * tool has three legal outcomes — a graded answer, a withheld one carrying
+ * guidance, and a reported failure — so a required field would fail two of the
+ * three. What the schema still buys is that a field the case reads is the type
+ * the case thinks it is.
+ */
+const Lookup = z.object({
+  answer: z.string().nullable().optional(),
+  sources: z.array(z.string()).optional(),
+  grounded: z.boolean().optional(),
+  answersTheQuestion: z.boolean().optional(),
+  rewrites: z.number().optional(),
+  guidance: z.string().optional(),
+  error: z.string().optional(),
+});
+
+/** Every `syncState` frame, in stream order. */
+function frames(session: EvalSession) {
+  return session
+    .events()
+    .flatMap((event) =>
+      event.type === "state.updated" ? [ProjectedSupport.parse(event.state)] : [],
+    );
+}
+
+/**
+ * What `answer_question` answered, in call order.
+ *
+ * `toolResultIn` over a ONE-CALL list per call: the name is that call's own, so
+ * the reader's "no such call" and "two calls" throws are unreachable and what is
+ * left is the parse, the schema, and its "never completed" failure — which is
+ * the half the hand-rolled `JSON.parse(call.result ?? "null")` this replaced got
+ * wrong, reading `null` for a tool that never came back.
+ */
+function lookups(session: EvalSession) {
+  return session
+    .toolCalls()
+    .filter((call) => call.name === "answer_question")
+    .map((call) => toolResultIn([call], call.name, Lookup));
+}
+
+describeEval(agentDef, (test) => {
+  test(
+    "nothing about the product is said without going through the knowledge base",
+    async ({ session }) => {
+      const turn = await session.say("How much notice do I have to give if I want to cancel?");
+
+      // The prompt's one rule: `answer_question` is the only route to anything
+      // about Meridian Fibre. A model that answered a notice period from its own
+      // knowledge would sound identical down a phone.
+      const asked = turn.toolCalls.filter((call) => call.name === "answer_question");
+      // The reply rides in the message, because the failure that matters here is
+      // a turn that SPOKE without looking anything up — "let me check that for
+      // you" and then nothing, or worse, a notice period from memory.
+      expect(
+        asked.length,
+        `tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
+      ).toBe(1);
+
+      for (const payload of lookups(session)) {
+        // Three legal outcomes, and the invariant that spans them: an answer the
+        // caller is given is an answer the hallucination grader passed. `grounded:
+        // false` reaching the model is the one thing the whole apparatus exists
+        // to prevent — and a broken lookup is REPORTED rather than worked around.
+        if (payload.error !== undefined) continue;
+        if (payload.answer === null) continue;
+        expect(payload.grounded, `ungrounded answer handed to the model: ${payload.answer}`).toBe(
+          true,
+        );
+      }
+    },
+    {
+      stubReply: [{ tool: "answer_question", args: { question: "notice to cancel" } }, "One sec."],
+    },
+  );
+
+  test(
+    "a logged ticket's callback number never leaves the server",
+    async ({ session }) => {
+      // Two turns, because that is the shape the prompt asks for: the lookup comes
+      // back empty, the line offers a ticket, and only then is there a number to
+      // take. Asked for in one breath, a live model reasonably answers the offer
+      // instead of acting on it.
+      await session.say("Can I add a pay-monthly mobile SIM to my broadband account?");
+      await session.say("Yes please, log that one — my callback number is 07700 900123.");
+
+      const logged = session.toolCalls().find((call) => call.name === "log_ticket");
+      expect(
+        logged,
+        `tools called: ${session
+          .toolCalls()
+          .map((c) => c.name)
+          .join(", ")}`,
+      ).toBeDefined();
+      expect(logged?.result).toMatch(/TCK\d{4}/);
+
+      const latest = frames(session).at(-1);
+      // The projection carries the REFERENCE and nothing else — that is why
+      // `supportView` exists rather than `syncState` taking the state itself.
+      expect(latest?.ticket).toMatch(/^TCK\d{4}$/);
+      for (const frame of frames(session)) {
+        expect(JSON.stringify(frame)).not.toMatch(/900123/);
+      }
+    },
+    {
+      stubReply: [
+        { tool: "answer_question", args: { question: "add a mobile SIM to my account" } },
+        "I don't have that documented. Shall I log a ticket for a callback?",
+        {
+          tool: "log_ticket",
+          args: { question: "add a mobile SIM to my account", callback: "07700 900123" },
+        },
+        "That's logged — the reference is T C K four thousand and one.",
+      ],
+    },
+  );
+
+  test(
+    "the answer comes from the document the caller asked about, not the one next to it",
+    async ({ session }) => {
+      // Short and single, deliberately: the claim here is which document the
+      // grader passed, so the question is phrased to measure that and not the
+      // loop's pacing.
+      //
+      // It used to say the corrective path could not fit the runtime's 30s
+      // per-tool deadline. Re-measured against the live gateway, that is no
+      // longer true by an order of magnitude — the two-attempt path runs in
+      // 2.8s of nine model calls — so the phrasing above is a choice about what
+      // this case measures rather than a way around a clock. The numbers and
+      // what they do and do not license are on `LOOKUP_BUDGET_MS` in
+      // `tools/answer_question.ts`.
+      const turn = await session.say("How much notice do I have to give to cancel my contract?");
+
+      const [payload] = lookups(session);
+      expect(
+        payload,
+        `tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
+      ).toBeDefined();
+      // The whole verdict in the message: `grounded: undefined` on its own does
+      // not say whether the lookup failed, or ran and refused.
+      const verdict = JSON.stringify(payload);
+      expect(payload?.grounded, verdict).toBe(true);
+      expect(payload?.answersTheQuestion, verdict).toBe(true);
+
+      const sources = payload?.sources ?? [];
+      expect(sources).toContain("Cancelling your contract");
+      // THE claim, and the reason the knowledge base is built the way it is:
+      // "cancelling your contract" and "cancelling an engineer visit" are two
+      // documents, two fees and one word apart. A retriever with no grader hands
+      // back both, and sixty pounds gets read down the phone as the answer to a
+      // contract question.
+      expect(sources).not.toContain("Cancelling or rebooking an engineer visit");
+      // Either spelling: the claim is that the notice period came out of D5, not
+      // how the answer prompt chose to write a numeral.
+      expect(String(payload?.answer)).toMatch(/thirty days|30 days/i);
+      expect(String(payload?.answer)).not.toMatch(/sixty pound|abortive/i);
+      expect(turn.text).not.toMatch(/sixty|abortive/i);
+    },
+    // Live only: the grader's verdict IS the measurement.
+    //
+    // THE CLOCK, and it is no longer the limit it was recorded as here.
+    // `answer_question` is up to eleven sequential model calls and the tool
+    // executor abandons any tool at `TOOL_EXECUTION_TIMEOUT_MS` (30s) — a
+    // deadline neither an agent author nor an eval can raise — so this comment
+    // used to say the two-attempt corrective path did not fit, which made this
+    // case a coin toss on the gateway rather than a reading of the grader.
+    // Re-measured: the whole loop is 2.0-3.1s, the corrective path 2.8s of nine
+    // calls, and eight consecutive live runs of this suite were green. The
+    // margin belongs to the service, not to us, so the tool keeps its own
+    // budget and this case keeps a question that settles on one attempt; if the
+    // gateway slows down again the failure says which, because a spent budget
+    // comes back as the graph's own exit sentence rather than as a timeout.
+    { live: true },
+  );
+
+  test(
+    "a question nothing covers is refused, after the corrective edge has tried",
+    async ({ session }) => {
+      const turn = await session.say(
+        "Can I add a pay-monthly mobile SIM with unlimited data to my broadband account?",
+      );
+
+      const [payload] = lookups(session);
+      expect(
+        payload,
+        `tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
+      ).toBeDefined();
+      // Withheld, not softened: `answer: null` is the tool refusing to hand the
+      // model something to read out, and the guidance is the exit the grading
+      // apparatus needs — a support line that can only answer will answer wrong.
+      // A lookup that BROKE and one that ran and found nothing are different
+      // findings, and neither is readable from "answer is not null".
+      expect(payload?.answer, JSON.stringify(payload)).toBeNull();
+      expect(String(payload?.guidance)).toMatch(/log_ticket/);
+      // The corrective edge really ran: nothing relevant means the RETRIEVAL was
+      // wrong, so the query is rewritten and retried before the loop gives up.
+      expect(payload?.rewrites).toBeGreaterThanOrEqual(1);
+
+      const trace = frames(session).at(-1)?.trace;
+      expect(trace?.exhausted).toBe(true);
+      expect(trace?.docs.every((doc) => !doc.relevant)).toBe(true);
+      expect(trace?.steps.map((step) => step.node)).toContain("transform_query");
+    },
+    { live: true },
+  );
+});