@alexkroman1/aai-cli 7.0.0 → 8.1.0

package/dist/templates/spoken-summary/agent.eval.test.ts

@@ -0,0 +1,342 @@
+// Copyright 2026 the AAI authors. MIT license.
+// An EVAL for a WORKFLOW APP: does the run actually do the work? Run it with
+// `aai eval`.
+//
+// `agent.test.ts` asserts about the declaration and drives the four legs one at
+// a time. This drives the WHOLE BODY — `spokenSummaryFlow` from the top — and
+// what it is here to check is the thing that is this template's whole reason to
+// exist and that no per-step spec can see: the AUDIO ROUND TRIP. A recording
+// goes in, and what comes out is a run whose OUTPUT NAMES A FILE that is really
+// in the store, put there by the same step that spoke it.
+//
+// `describeWorkflowEval` picks the providers for you and says which it picked:
+//
+//   * with `ASSEMBLYAI_API_KEY` — a LIVE run. The `{ live: true }` case below
+//     downloads a real four-minute news clip, really transcribes it and really
+//     summarizes it. That spends money and about a minute.
+//   * without one — a SCRIPTED run: the same body, the same four legs, with the
+//     provider answered in memory.
+//
+// Three of the four cases are SCRIPTED IN BOTH MODES, deliberately. Their claims
+// are about WIRING — that the id in the output is the id of the file that was
+// written, that the voice was handed the script and not the bullet list, that
+// the poll loop asks for its wait — and a live provider can neither confirm nor
+// deny any of them: it can only be asked and then have its answer accepted,
+// which is not evidence. The live case is the one that answers "is the summary
+// really of this recording".
+//
+// The UPLOAD STORE is faked in both modes, and that is not a mode question: it
+// is this app's own storage (a database row and a blob), not a provider, and an
+// eval has neither. `{ writable: true }` is what lets the store accept the
+// write — opt-in precisely so a step that stored a file nobody meant it to
+// still fails.
+//
+// TWO PROVIDER LEGS ARE NOT REACHABLE LIVE from an eval today, and both are
+// named where they are worked around rather than left as a surprise:
+//
+//   * the streaming UPLOAD leg needs a `stepFetch` — see `liveStepFetch` below.
+//   * `stepSpeak` needs a synthesizer, and the engine publishes none by default
+//     while `@alexkroman1/aai-runtime` exports no real one to pass. So the VOICE
+//     is faked in every case here, live included, and the live case's claim is
+//     about the transcript and the summary rather than about audible audio.
+//
+// WHAT NO EVAL HERE COVERS: durability. Imported through vitest with no bundler
+// in the path, a `"use workflow"` body is an ordinary async function — no
+// journal, no replay, and no per-step retry, so a rate-limited live run FAILS
+// where a deployed one would have ridden it out, and the resume-replays-the-id
+// property that makes speak-and-store ONE step is argued here rather than
+// exercised. `run.slept` below is the other half of that admission written as
+// an assertion. `aai-cli`'s `dev-workflow.scenario.test.ts` is the tier that
+// really suspends and resumes a run.
+import {
+  installStubSpeech,
+  installStubTranscribe,
+  installStubUploads,
+} from "@alexkroman1/aai/testing/vitest";
+import { describeWorkflowEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import agentDef, { spokenSummary } from "./agent.ts";
+import { POLL_INTERVAL } from "./workflows/transcribe.ts";
+
+/** The id every case uploads the recording under. */
+const UPLOAD_ID = "upl_eval";
+
+/** The public sample recording — four and a half minutes of real speech. */
+const LIVE_RECORDING = "https://assembly.ai/wildfires.mp3";
+
+/** What a scripted transcript says, so a summary of it is checkable. */
+const TRANSCRIPT =
+  "Right, standup. The launch is on for Tuesday the fourth. Two bugs are left in " +
+  "checkout, both assigned to Priya, and neither is a blocker. Marketing wants the " +
+  "blog post by Monday. If the second bug slips we ship anyway and patch on Wednesday.";
+
+/**
+ * The reply the model is scripted to give.
+ *
+ * The `spoken` script deliberately shares NO wording with `points`, which is
+ * what makes "the voice was handed the script" an assertion rather than a
+ * coincidence — see the second case.
+ */
+const REPLY = {
+  headline: "Launch is on for Tuesday",
+  points: ["Ship Tuesday the fourth", "Two checkout bugs, neither blocking", "Blog post by Monday"],
+  spoken:
+    "Everything is lined up for the fourth. A couple of small things are still open " +
+    "in checkout and Priya has both of them; nothing there is holding the release. " +
+    "Marketing needs the write-up at the start of the week.",
+};
+
+/**
+ * A `stepFetch` for the LIVE legs, and it exists to work around a gap rather
+ * than to add anything.
+ *
+ * The eval engine publishes no `stepFetch`, so a step's HTTP falls back to
+ * `globalThis.fetch` — which cannot send this app's UPLOAD leg at all.
+ * `stepTranscribeUpload` streams a stored recording window by window, and an
+ * iterable body requires `duplex: "half"`; the published fetch adds it (see
+ * `sdk/step-fetch.ts`, which says so) and the fallback does not, so a live run
+ * dies on `RequestInit: duplex option is required when sending a body` before
+ * the provider is ever reached. `EvalWorkflowsOptions.stepFetch` is the
+ * documented seam for a host to supply its own, and this is the smallest one
+ * that works. Every scripted case below REPLACES it, publishing being a
+ * replacement.
+ */
+const liveStepFetch = (url: string, init: Record<string, unknown> = {}): Promise<Response> =>
+  globalThis.fetch(url, { ...init, duplex: "half" } as RequestInit);
+
+/** Publish this app's own store, writable because the last step writes to it. */
+function publish(bytes: Uint8Array, name: string, type: string) {
+  return installStubUploads({ [UPLOAD_ID]: { bytes, name, type } }, { writable: true });
+}
+
+/**
+ * Answer every leg of the run in memory: the three transcription calls, and the
+ * model.
+ *
+ * ONE fake, because publishing a `stepFetch` REPLACES — a flow that transcribes
+ * AND calls a model cannot install two, which is exactly what `otherwise` is
+ * for. The transcription half is the SDK's own fake rather than this file's
+ * hand-typed wire: it routes off the SDK's endpoint constants, so a case cannot
+ * pass because the fake and the step agree on a typo.
+ */
+function scriptProvider(options: { text?: string; pendingPolls?: number } = {}) {
+  return installStubTranscribe({
+    text: options.text ?? TRANSCRIPT,
+    durationSec: 42,
+    // Passed straight through rather than conditionally spread: the option
+    // already admits `undefined`, and `guard-invariants` rule 2 counts the
+    // spread.
+    pendingPolls: options.pendingPolls,
+    otherwise: (request) =>
+      request.url.includes("llm-gateway")
+        ? { body: { choices: [{ message: { content: JSON.stringify(REPLY) } }] } }
+        : undefined,
+  });
+}
+
+describeWorkflowEval(
+  agentDef,
+  (test) => {
+    test("the run's output NAMES a file that one step spoke and stored", async ({ app }) => {
+      // Scripted in both modes: the claim is that the id in the output is the id
+      // of the file in the store, and no provider can be asked to make that true.
+      // This is the case that catches speak-and-store coming apart — a run that
+      // returned bytes, or an id nothing wrote, or two ids because the synthesis
+      // and the store became two steps.
+      const uploads = publish(new Uint8Array(64), "standup.wav", "audio/wav");
+      const provider = scriptProvider();
+      const speech = installStubSpeech({ pcmBytes: 96_000 });
+
+      const run = await app.run(spokenSummary, { recording: UPLOAD_ID });
+
+      // The error FIRST, so a failed run names its own reason instead of
+      // reporting "expected 'failed' to be 'completed'".
+      expect(run.error).toBeUndefined();
+      expect(run.status).toBe("completed");
+      const output = run.output;
+      if (output === undefined) expect.fail("a completed run must carry an output");
+
+      // EXACTLY one write. Two would mean the synthesis and the store had come
+      // apart into two steps, which is the mistake this template exists to argue
+      // against — a step is journaled by its return value, so an id replays and
+      // bytes do not.
+      expect(uploads.writes).toHaveLength(1);
+      const written = uploads.writes[0];
+      if (written === undefined) expect.fail("the speaking step must have stored a file");
+      // The output carries the ID of that write, and an id is a string — not the
+      // audio. A run's output is read back as JSON.
+      expect(output.audio).toBe(written.id);
+      expect(typeof output.audio).toBe("string");
+
+      // And the bytes are really there, and are really a WAV: named and typed for
+      // a browser, because the byte route serves what it was given and nothing
+      // plays a file handed to it as octet-stream.
+      expect(written.name).toBe("summary.wav");
+      expect(written.type).toBe("audio/wav");
+      expect(written.bytes.byteLength).toBe(44 + 96_000);
+      expect(String.fromCharCode(...written.bytes.subarray(0, 4))).toBe("RIFF");
+      expect(String.fromCharCode(...written.bytes.subarray(8, 12))).toBe("WAVE");
+      // 96,000 bytes at 24 kHz mono 16-bit is two seconds, which is what the
+      // page prints next to the player.
+      expect(output.audioDurationMs).toBe(2000);
+
+      // The rest of the round trip, so a run that stored audio for the wrong text
+      // is not mistaken for a working one.
+      expect(output.source).toBe("standup.wav");
+      expect(output.durationMs).toBe(42_000);
+      expect(output.transcript).toBe(TRANSCRIPT);
+      expect(output.points).toHaveLength(3);
+      expect(speech.calls).toHaveLength(1);
+
+      // Four legs, four narrated lines, in order — which is what a page watching
+      // the run renders.
+      expect(run.reported[0]).toMatch(/^Uploading standup\.wav /);
+      expect(run.reported).toContain("Summarizing the transcript.");
+      expect(run.reported.at(-1)).toMatch(/^Recorded a 2s summary in \w+'s voice\.$/);
+      // The job finished on its first poll, so no durable wait was asked for.
+      expect(run.slept).toEqual([]);
+      // One upload of the recording, and one submit — the split that exists so a
+      // fault in the submit does not re-upload the file.
+      expect(provider.calls.filter((call) => call.leg === "upload")).toHaveLength(1);
+      expect(provider.calls.filter((call) => call.leg === "submit")).toHaveLength(1);
+    });
+
+    test("the voice reads the SCRIPT the model was asked for, not the points", async ({ app }) => {
+      // Scripted in both modes: what a live voice says is not evidence about
+      // which string it was handed. This is the case that catches the template's
+      // central prompt decision regressing — synthesize the bullet list and you
+      // get a voice reading "one. two. three." with no connective tissue.
+      publish(new Uint8Array(64), "standup.wav", "audio/wav");
+      const provider = scriptProvider();
+      const speech = installStubSpeech();
+
+      const run = await app.run(spokenSummary, { recording: UPLOAD_ID, voice: "michael" });
+
+      expect(run.error).toBeUndefined();
+      const spokenText = speech.calls[0]?.text;
+      expect(spokenText).toBe(REPLY.spoken);
+      // The script and the points share no wording, so this is the assertion:
+      // whatever was spoken, it was not the list.
+      for (const point of REPLY.points) expect(spokenText).not.toContain(point);
+      // The form's choice really reaches the synthesizer.
+      expect(speech.calls[0]?.voice).toBe("michael");
+      // Both shapes survive to the output — one to read, one that was heard.
+      expect(run.output?.points).toEqual(REPLY.points);
+      expect(run.output?.spoken).toBe(REPLY.spoken);
+
+      // And the model was ASKED for both, over the transcript it was given. A
+      // prompt that stopped asking for a script is how the field goes missing.
+      const prompt = String(provider.calls.find((call) => call.leg === "other")?.body ?? "");
+      expect(prompt).toContain("READ ALOUD");
+      expect(prompt).toContain("The launch is on for Tuesday the fourth");
+    });
+
+    test("a recording with no speech stops before the model and the voice", async ({ app }) => {
+      // Scripted in both modes, and it costs nothing live either way: silence
+      // transcribes SUCCESSFULLY to nothing, so without a terminal failure here
+      // the run would go on to summarize no words and store half a second of
+      // audio — a green run with an empty product.
+      const uploads = publish(new Uint8Array(64), "silence.wav", "audio/wav");
+      const provider = scriptProvider({ text: "   " });
+      const speech = installStubSpeech();
+
+      const run = await app.run(spokenSummary, { recording: UPLOAD_ID });
+
+      expect(run.status).toBe("failed");
+      expect(run.error).toMatch(/no speech in that recording/i);
+      expect(run.output).toBeUndefined();
+      // Nothing was summarized and nothing was spoken, which is the half that
+      // makes this more than an error-message assertion.
+      expect(provider.calls.filter((call) => call.leg === "other")).toEqual([]);
+      expect(speech.calls).toEqual([]);
+      expect(uploads.writes).toEqual([]);
+      expect(run.reported).not.toContain("Summarizing the transcript.");
+    });
+
+    test("an unfinished job is waited out with a DURABLE sleep, not a busy loop", async ({
+      app,
+    }) => {
+      // Scripted in both modes: a live job cannot be asked to stay queued for
+      // exactly two polls, and this is the one place the poll loop's shape is
+      // visible — one submit, one poll per round, and a recorded wait between
+      // them. A loop that re-submitted, or one that spun with no wait, both
+      // produce a correct transcript and a wrong bill.
+      publish(new Uint8Array(64), "standup.wav", "audio/wav");
+      const provider = scriptProvider({ pendingPolls: 2 });
+      installStubSpeech();
+
+      const run = await app.run(spokenSummary, { recording: UPLOAD_ID });
+
+      expect(run.error).toBeUndefined();
+      expect(run.output?.transcript).toBe(TRANSCRIPT);
+      // Two waits for three polls: asked for and — this being an eval rather
+      // than a deployment — recorded rather than taken.
+      expect(run.slept).toEqual([{ duration: POLL_INTERVAL }, { duration: POLL_INTERVAL }]);
+      expect(provider.calls.filter((call) => call.leg === "poll")).toHaveLength(3);
+      // The expensive half happened ONCE, which is the whole reason the upload
+      // and the submit are separate steps.
+      expect(provider.calls.filter((call) => call.leg === "upload")).toHaveLength(1);
+      expect(provider.calls.filter((call) => call.leg === "submit")).toHaveLength(1);
+    });
+
+    test(
+      "really transcribes and summarizes a real recording",
+      async ({ app }) => {
+        // LIVE ONLY, and the one case here that measures rather than checks
+        // wiring. Three things have to be real for it to pass: the file has to
+        // stream to the provider, the transcript has to be of THAT recording, and
+        // the model has to write a script from that transcript.
+        //
+        // The VOICE is faked even here, and that is a harness limit rather than a
+        // choice — `stepSpeak` reads a published synthesizer, the eval engine
+        // publishes none, and no real one is exported to pass it. So what this
+        // case does NOT claim is that the audio is audible; the round trip's
+        // structure (one write, an id in the output, real WAV framing) is what
+        // the first case pins, in memory, where it is a fact rather than a hope.
+        const speech = installStubSpeech({ pcmBytes: 48_000 });
+        const response = await fetch(LIVE_RECORDING);
+        expect(response.ok).toBe(true);
+        const mp3 = new Uint8Array(await response.arrayBuffer());
+        const uploads = publish(mp3, "wildfires.mp3", "audio/mpeg");
+
+        const run = await app.run(spokenSummary, { recording: UPLOAD_ID });
+
+        expect(run.error).toBeUndefined();
+        expect(run.status).toBe("completed");
+        const output = run.output;
+        if (output === undefined) expect.fail("a completed run must carry an output");
+
+        // The transcript is of THIS recording — a news segment about smoke from
+        // Canadian wildfires reaching the US east coast.
+        expect(output.transcript).toMatch(/wildfire/i);
+        expect(output.transcript).toMatch(/canada/i);
+        expect(output.words).toBeGreaterThan(400);
+        // The FILENAME a reader sees is the one they uploaded, not the opaque id.
+        expect(output.source).toBe("wildfires.mp3");
+        expect(output.durationMs).toBeGreaterThan(250_000);
+
+        // The summary is of the transcript, not of summarizing in general.
+        const written = `${output.headline} ${output.points.join(" ")} ${output.spoken}`;
+        expect(written).toMatch(/smoke|wildfire|air/i);
+        expect(output.points.length).toBeGreaterThan(0);
+        expect(output.points.length).toBeLessThanOrEqual(4);
+        // A SCRIPT rather than a list: sentences, no bullet markers, and long
+        // enough that a voice reading it has something to say. This is the
+        // template's central prompt decision, measured against a real model.
+        expect(output.spoken).toMatch(/[.!?]/);
+        expect(output.spoken).not.toMatch(/^\s*[-*\u2022]/m);
+        expect(output.spoken.length).toBeGreaterThan(80);
+        // And it is the SCRIPT that was spoken, not the points a page renders.
+        expect(speech.calls[0]?.text).toBe(output.spoken);
+
+        // The store really holds what the output names.
+        expect(uploads.writes).toHaveLength(1);
+        const stored = uploads.read(output.audio);
+        if (stored === undefined) expect.fail("the run's `audio` id must name a stored file");
+        expect(String.fromCharCode(...stored.bytes.subarray(0, 4))).toBe("RIFF");
+      },
+      { live: true },
+    );
+  },
+  { stepFetch: liveStepFetch },
+);

package/dist/templates/support-line/agent.eval.test.ts

@@ -0,0 +1,273 @@
+/// <reference types="vite/client" />
+
+// An EVAL: does the line really refuse to say what it cannot ground? Run it
+// with `aai eval`.
+//
+// `agent.test.ts` drives the retriever and each CRAG node directly. What it
+// cannot see is the claim this template exists to make — that a MODEL holding
+// this tool list cannot state a fee it did not just retrieve. So these cases
+// drive a real session and read the verdicts off the tool results: which
+// documents the grader passed, whether the answer was grounded, and whether an
+// answer it could not ground was WITHHELD.
+//
+// **One thing a SCRIPTED run cannot reach: `ctx.generate` with a SCHEMA.** The
+// stub model registers as an LLM provider, so `ctx.generate` resolves it, and it
+// answers a plain text generation from the script. Every node in this template's
+// loop passes a `schema` though (`binaryScore`), and that overload comes back
+// `{"error":"The knowledge base lookup failed: No output generated."}` — so the
+// whole corrective loop is unreachable scripted. That is the template's central
+// tool, so the two cases that run without a key are written as invariants both
+// modes satisfy (a failed lookup is REPORTED, never answered around), and the
+// two that read a real verdict are `{ live: true }`.
+//
+// **`system-prompt.md` is applied HERE, not by `agent.ts`.** The build discovers
+// the file, so an eval driving the raw default export would run this agent with
+// the FRAMEWORK DEFAULT prompt — and the discipline that prompt imposes is the
+// entire subject of this file. Measured against the default: the model answers
+// product questions from its own knowledge and skips the tools the prompt exists
+// to route it through, so a case run that way measures nothing it claims to.
+
+import { withSystemPrompt } from "@alexkroman1/aai/manifest";
+import { withDiscoveredTools } from "@alexkroman1/aai/testing";
+import { type EvalSession, toolResultIn } from "@alexkroman1/aai-runtime/eval";
+import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import { z } from "zod";
+import authoredAgent from "./agent.ts";
+import systemPrompt from "./system-prompt.md?raw";
+
+/** The def a DEPLOYED agent runs — see `agent.test.ts` on why the glob is here. */
+const agentDef = withSystemPrompt(
+  withDiscoveredTools(authoredAgent, import.meta.glob("./tools/*.ts", { eager: true })),
+  systemPrompt,
+);
+
+/**
+ * What the BROWSER is sent, as this eval reads it.
+ *
+ * Parsed rather than cast: `state.updated` carries `unknown`, so a projection
+ * that stopped matching FAILS naming the field, where the cast this replaced
+ * handed the assertions `undefined` and failed a line later on something else.
+ * It names only the fields asserted below, so `supportView` may grow without
+ * touching this.
+ */
+const ProjectedSupport = z.object({
+  ticket: z.string().nullable(),
+  trace: z
+    .object({
+      exhausted: z.boolean(),
+      docs: z.array(z.object({ relevant: z.boolean() })),
+      steps: z.array(z.object({ node: z.string() })),
+    })
+    .nullable(),
+});
+
+/**
+ * What `answer_question` answered, validated.
+ *
+ * All-optional, and that is the shape of the CONTRACT rather than laxness: the
+ * tool has three legal outcomes — a graded answer, a withheld one carrying
+ * guidance, and a reported failure — so a required field would fail two of the
+ * three. What the schema still buys is that a field the case reads is the type
+ * the case thinks it is.
+ */
+const Lookup = z.object({
+  answer: z.string().nullable().optional(),
+  sources: z.array(z.string()).optional(),
+  grounded: z.boolean().optional(),
+  answersTheQuestion: z.boolean().optional(),
+  rewrites: z.number().optional(),
+  guidance: z.string().optional(),
+  error: z.string().optional(),
+});
+
+/** Every `syncState` frame, in stream order. */
+function frames(session: EvalSession) {
+  return session
+    .events()
+    .flatMap((event) =>
+      event.type === "state.updated" ? [ProjectedSupport.parse(event.state)] : [],
+    );
+}
+
+/**
+ * What `answer_question` answered, in call order.
+ *
+ * `toolResultIn` over a ONE-CALL list per call: the name is that call's own, so
+ * the reader's "no such call" and "two calls" throws are unreachable and what is
+ * left is the parse, the schema, and its "never completed" failure — which is
+ * the half the hand-rolled `JSON.parse(call.result ?? "null")` this replaced got
+ * wrong, reading `null` for a tool that never came back.
+ */
+function lookups(session: EvalSession) {
+  return session
+    .toolCalls()
+    .filter((call) => call.name === "answer_question")
+    .map((call) => toolResultIn([call], call.name, Lookup));
+}
+
+describeEval(agentDef, (test) => {
+  test(
+    "nothing about the product is said without going through the knowledge base",
+    async ({ session }) => {
+      const turn = await session.say("How much notice do I have to give if I want to cancel?");
+
+      // The prompt's one rule: `answer_question` is the only route to anything
+      // about Meridian Fibre. A model that answered a notice period from its own
+      // knowledge would sound identical down a phone.
+      const asked = turn.toolCalls.filter((call) => call.name === "answer_question");
+      // The reply rides in the message, because the failure that matters here is
+      // a turn that SPOKE without looking anything up — "let me check that for
+      // you" and then nothing, or worse, a notice period from memory.
+      expect(
+        asked.length,
+        `tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
+      ).toBe(1);
+
+      for (const payload of lookups(session)) {
+        // Three legal outcomes, and the invariant that spans them: an answer the
+        // caller is given is an answer the hallucination grader passed. `grounded:
+        // false` reaching the model is the one thing the whole apparatus exists
+        // to prevent — and a broken lookup is REPORTED rather than worked around.
+        if (payload.error !== undefined) continue;
+        if (payload.answer === null) continue;
+        expect(payload.grounded, `ungrounded answer handed to the model: ${payload.answer}`).toBe(
+          true,
+        );
+      }
+    },
+    {
+      stubReply: [{ tool: "answer_question", args: { question: "notice to cancel" } }, "One sec."],
+    },
+  );
+
+  test(
+    "a logged ticket's callback number never leaves the server",
+    async ({ session }) => {
+      // Two turns, because that is the shape the prompt asks for: the lookup comes
+      // back empty, the line offers a ticket, and only then is there a number to
+      // take. Asked for in one breath, a live model reasonably answers the offer
+      // instead of acting on it.
+      await session.say("Can I add a pay-monthly mobile SIM to my broadband account?");
+      await session.say("Yes please, log that one — my callback number is 07700 900123.");
+
+      const logged = session.toolCalls().find((call) => call.name === "log_ticket");
+      expect(
+        logged,
+        `tools called: ${session
+          .toolCalls()
+          .map((c) => c.name)
+          .join(", ")}`,
+      ).toBeDefined();
+      expect(logged?.result).toMatch(/TCK\d{4}/);
+
+      const latest = frames(session).at(-1);
+      // The projection carries the REFERENCE and nothing else — that is why
+      // `supportView` exists rather than `syncState` taking the state itself.
+      expect(latest?.ticket).toMatch(/^TCK\d{4}$/);
+      for (const frame of frames(session)) {
+        expect(JSON.stringify(frame)).not.toMatch(/900123/);
+      }
+    },
+    {
+      stubReply: [
+        { tool: "answer_question", args: { question: "add a mobile SIM to my account" } },
+        "I don't have that documented. Shall I log a ticket for a callback?",
+        {
+          tool: "log_ticket",
+          args: { question: "add a mobile SIM to my account", callback: "07700 900123" },
+        },
+        "That's logged — the reference is T C K four thousand and one.",
+      ],
+    },
+  );
+
+  test(
+    "the answer comes from the document the caller asked about, not the one next to it",
+    async ({ session }) => {
+      // Short and single, deliberately: the claim here is which document the
+      // grader passed, so the question is phrased to measure that and not the
+      // loop's pacing.
+      //
+      // It used to say the corrective path could not fit the runtime's 30s
+      // per-tool deadline. Re-measured against the live gateway, that is no
+      // longer true by an order of magnitude — the two-attempt path runs in
+      // 2.8s of nine model calls — so the phrasing above is a choice about what
+      // this case measures rather than a way around a clock. The numbers and
+      // what they do and do not license are on `LOOKUP_BUDGET_MS` in
+      // `tools/answer_question.ts`.
+      const turn = await session.say("How much notice do I have to give to cancel my contract?");
+
+      const [payload] = lookups(session);
+      expect(
+        payload,
+        `tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
+      ).toBeDefined();
+      // The whole verdict in the message: `grounded: undefined` on its own does
+      // not say whether the lookup failed, or ran and refused.
+      const verdict = JSON.stringify(payload);
+      expect(payload?.grounded, verdict).toBe(true);
+      expect(payload?.answersTheQuestion, verdict).toBe(true);
+
+      const sources = payload?.sources ?? [];
+      expect(sources).toContain("Cancelling your contract");
+      // THE claim, and the reason the knowledge base is built the way it is:
+      // "cancelling your contract" and "cancelling an engineer visit" are two
+      // documents, two fees and one word apart. A retriever with no grader hands
+      // back both, and sixty pounds gets read down the phone as the answer to a
+      // contract question.
+      expect(sources).not.toContain("Cancelling or rebooking an engineer visit");
+      // Either spelling: the claim is that the notice period came out of D5, not
+      // how the answer prompt chose to write a numeral.
+      expect(String(payload?.answer)).toMatch(/thirty days|30 days/i);
+      expect(String(payload?.answer)).not.toMatch(/sixty pound|abortive/i);
+      expect(turn.text).not.toMatch(/sixty|abortive/i);
+    },
+    // Live only: the grader's verdict IS the measurement.
+    //
+    // THE CLOCK, and it is no longer the limit it was recorded as here.
+    // `answer_question` is up to eleven sequential model calls and the tool
+    // executor abandons any tool at `TOOL_EXECUTION_TIMEOUT_MS` (30s) — a
+    // deadline neither an agent author nor an eval can raise — so this comment
+    // used to say the two-attempt corrective path did not fit, which made this
+    // case a coin toss on the gateway rather than a reading of the grader.
+    // Re-measured: the whole loop is 2.0-3.1s, the corrective path 2.8s of nine
+    // calls, and eight consecutive live runs of this suite were green. The
+    // margin belongs to the service, not to us, so the tool keeps its own
+    // budget and this case keeps a question that settles on one attempt; if the
+    // gateway slows down again the failure says which, because a spent budget
+    // comes back as the graph's own exit sentence rather than as a timeout.
+    { live: true },
+  );
+
+  test(
+    "a question nothing covers is refused, after the corrective edge has tried",
+    async ({ session }) => {
+      const turn = await session.say(
+        "Can I add a pay-monthly mobile SIM with unlimited data to my broadband account?",
+      );
+
+      const [payload] = lookups(session);
+      expect(
+        payload,
+        `tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
+      ).toBeDefined();
+      // Withheld, not softened: `answer: null` is the tool refusing to hand the
+      // model something to read out, and the guidance is the exit the grading
+      // apparatus needs — a support line that can only answer will answer wrong.
+      // A lookup that BROKE and one that ran and found nothing are different
+      // findings, and neither is readable from "answer is not null".
+      expect(payload?.answer, JSON.stringify(payload)).toBeNull();
+      expect(String(payload?.guidance)).toMatch(/log_ticket/);
+      // The corrective edge really ran: nothing relevant means the RETRIEVAL was
+      // wrong, so the query is rewritten and retried before the loop gives up.
+      expect(payload?.rewrites).toBeGreaterThanOrEqual(1);
+
+      const trace = frames(session).at(-1)?.trace;
+      expect(trace?.exhausted).toBe(true);
+      expect(trace?.docs.every((doc) => !doc.relevant)).toBe(true);
+      expect(trace?.steps.map((step) => step.node)).toContain("transform_query");
+    },
+    { live: true },
+  );
+});

package/dist/templates/support-line/system-prompt.md

@@ -7,7 +7,8 @@ The one rule that matters:
 
 Running a call:
 
-- The caller is waiting while the lookup runs, so say a short "let me check that for you" before calling `answer_question` — never in silence.
+- **Every turn in which the caller asks anything about Meridian Fibre begins with a call to `answer_question`.** The call goes out first, in that turn, before you decide what to say. That is the whole procedure and there is no version of it where the lookup happens later.
+- "Let me check that for you" is not a check. It is what you may say while the call you have already made is running, in the same turn — five words, no more. **A turn that ends on that sentence with no `answer_question` call in it has hung up on the caller**: they are holding a line that never comes back, which is worse than the silence the sentence exists to fill. If you have said it, the tool call is already out or the turn is not finished.
 - Give the answer it returns in your own words, keeping every number exactly as it came back.
 - When it returns no answer, say plainly that you do not have that documented. Do not guess, and do not soften it into a maybe. Offer to log a ticket.
 - When it returns an answer with a caveat, give the answer and then offer the ticket.