@alexkroman1/aai-cli 7.0.0 → 8.1.0

package/dist/templates/dispatch-center/agent.eval.test.ts

@@ -0,0 +1,349 @@
+/// <reference types="vite/client" />
+
+import { withSystemPrompt } from "@alexkroman1/aai/manifest";
+import type { SessionEvent } from "@alexkroman1/aai/protocol";
+import { withDiscoveredTools } from "@alexkroman1/aai/testing";
+// An EVAL: does the desk actually behave? Run it with `aai eval`.
+//
+// `agent.test.ts` drives each tool directly. What it cannot ask is whether the
+// AGENT — a model, reading this system prompt, holding these twelve tools —
+// works the call in the right ORDER, which for a dispatch desk is the whole
+// product. Four claims, each one a MECHANISM this template is built out of:
+//
+//   1. nothing can be dispatched before a call has been logged,
+//   2. a 911 call is logged, scored, and the flow moves to triaging,
+//   3. units really roll, and the position follows them to monitoring,
+//   4. a unit already on a call is not sent to a second one.
+//
+// So every assertion reads the mechanism's own output — the dialog gate's
+// refusal, the tool result, and the dashboard the browser is sent — rather
+// than judging the sentence the model chose to say.
+//
+// What no eval here can see: anything below the audio boundary. Whether a
+// dispatcher reading a callsign in bursts lands as one turn is a property of
+// endpointing, and these fake speech stages remove it.
+import { type EvalSession, type EvalTurn, lastStateIn } from "@alexkroman1/aai-runtime/eval";
+import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import { z } from "zod";
+import authoredAgent from "./agent.ts";
+import systemPrompt from "./system-prompt.md?raw";
+
+/**
+ * The def a DEPLOYED agent runs: authored, plus what `tools/` declares, plus
+ * what `system-prompt.md` says.
+ *
+ * BOTH wrappers are load-bearing and neither is applied by `agent()` — they are
+ * applied by the BUILD (`aai build` enumerates `tools/` and resolves the prompt
+ * file), so an eval driving the raw default export would measure a twelve-tool
+ * desk with no tools and the FRAMEWORK DEFAULT prompt. For this template that
+ * is the whole subject: "location is always the first priority", the radio
+ * style and "never leave a critical incident without a resource" are all in
+ * that file.
+ *
+ * The glob is written here rather than reached for from a shared helper because
+ * this file SHIPS — it is what a scaffolded project runs.
+ */
+const dispatchAgent = withSystemPrompt(
+  withDiscoveredTools(authoredAgent, import.meta.glob("./tools/*.ts", { eager: true })),
+  systemPrompt,
+);
+
+/** The six tools gated on `working` — the ones that refuse until something has
+ *  been logged. Listed here so ADDING an ungated mutating tool is a deliberate
+ *  edit to this file rather than a silent gap. */
+const GATED_TOOLS = new Set([
+  "incident_triage",
+  "incident_add_note",
+  "incident_escalate",
+  "incident_update_status",
+  "resources_dispatch",
+  "resources_update_status",
+]);
+
+/** The first incident of a session. `createIncident` counts from 1 per slot,
+ *  and a slot is per session, so this id is deterministic. */
+const FIRST_INCIDENT = "INC-0001";
+/** The second, for the case that logs two. */
+const SECOND_INCIDENT = "INC-0002";
+
+/**
+ * What the BROWSER is sent, as this eval reads it.
+ *
+ * Parsed rather than cast: `state.updated` carries `unknown`, and a schema that
+ * stops matching is a loud failure naming the field where a cast would hand the
+ * assertions `undefined` and fail three lines later. It names only the fields
+ * asserted below, so `dashboardView` may grow without touching this.
+ */
+const ProjectedDashboard = z.object({
+  systemAlertLevel: z.string(),
+  incidents: z.array(
+    z.object({
+      id: z.string(),
+      severity: z.string(),
+      status: z.string(),
+      location: z.string(),
+    }),
+  ),
+});
+
+/**
+ * The latest dashboard the session pushed, or undefined if it pushed none.
+ *
+ * `lastStateIn` is the SDK's reader, and the schema is what it is worth passing:
+ * a frame that stopped matching FAILS naming the field, where the cast this
+ * replaced would have handed the assertions `undefined`.
+ */
+const dashboard = (events: readonly SessionEvent[]) => lastStateIn(events, ProjectedDashboard);
+
+/** One incident as the browser sees it. */
+const boardEntry = (events: readonly SessionEvent[], id: string) =>
+  dashboard(events)?.incidents.find((i) => i.id === id);
+
+/**
+ * The dialog gate's own refusal sentence, for the state it names.
+ *
+ * The character class absorbs the JSON escaping: a tool result reaches the
+ * event stream as a serialized string, so the state name arrives inside
+ * `\\"standby\\"` rather than plain quotes.
+ */
+const refusalAt = (state: string) =>
+  new RegExp(`Not available yet: this conversation is at [\\\\"]*${state}`);
+
+/**
+ * Drive a whole shift, one dispatcher line at a time, and hand back every turn.
+ *
+ * The cases below assert about the turn a MECHANISM fired in rather than about
+ * turn number two: how many turns a desk takes to get there is the model's
+ * business and it varies, and a case pinned to a turn index is a flake with a
+ * misleading name.
+ */
+async function sayAll(session: EvalSession, lines: readonly string[]): Promise<EvalTurn[]> {
+  const turns: EvalTurn[] = [];
+  for (const line of lines) turns.push(await session.say(line));
+  return turns;
+}
+
+/** Every call to `tool` across the whole shift. */
+const callsTo = (session: EvalSession, tool: string) =>
+  session.toolCalls().filter((c) => c.name === tool);
+
+/** The 911 call all four cases open with — "cardiac arrest" is what
+ *  `recommendSeverity` reads as critical and `recommendType` as medical. */
+const THE_CALL =
+  "Take this call: 400 Oak Street, cardiac arrest, CPR in progress. " +
+  "Caller is Jane Doe, callback five five five, zero one zero one.";
+
+/** The scripted turn that logs it. */
+const LOG_TURN = [
+  {
+    tool: "incident_create",
+    args: {
+      location: "400 Oak Street",
+      description: "Cardiac arrest, CPR in progress",
+      callerName: "Jane Doe",
+      callerPhone: "555-0101",
+    },
+  },
+  "Copy — logged as a priority one. Confirming severity and getting units rolling.",
+] as const;
+
+describeEval(dispatchAgent, (test) => {
+  test(
+    "refuses a unit's radio call while nothing is logged on the shift",
+    async ({ session }) => {
+      // A unit radioing in is not an emergency, so it gives the desk nothing to
+      // log — which is what makes the gate the only thing that can answer.
+      // Measured live: a DISPATCH order does not, because a competent model logs
+      // the call first and then legitimately reaches the gated tools from
+      // `working`, and a case that read that as a gate failure would be wrong.
+      const turn = await session.say("Medic-1 just radioed in — mark them available.");
+
+      // Every gated call the model made while the shift was still in standby
+      // has to have been refused, and the refusal has to say where the shift
+      // actually is. The `incident_create` guard is the lesson above, kept:
+      // anything issued AFTER a call was logged is in `working` and legal.
+      const logged = turn.toolCalls.findIndex((c) => c.name === "incident_create");
+      for (const [index, call] of turn.toolCalls.entries()) {
+        if (!GATED_TOOLS.has(call.name)) continue;
+        if (logged !== -1 && index > logged) continue;
+        expect(call.result).toMatch(refusalAt("standby"));
+      }
+      // Nothing was logged, so the board the browser holds is still empty.
+      expect(dashboard(session.events())?.incidents ?? []).toEqual([]);
+      // And it said so rather than pretending the unit had been updated.
+      expect(turn.text).toMatch(/nothing|no incident|not logged|standby|call|log/i);
+    },
+    {
+      stubReply: [
+        { tool: "resources_update_status", args: { callsign: "Medic-1", status: "available" } },
+        "Nothing is logged on this shift yet — give me a call and I'll open it.",
+      ],
+    },
+  );
+
+  test(
+    "logs a 911 call, scores it, and moves the shift to triaging",
+    async ({ session }) => {
+      const turn = await session.say(THE_CALL);
+
+      const logged = turn.toolCalls.find((c) => c.name === "incident_create");
+      expect(logged?.args.location).toMatch(/oak/i);
+      // The recommendation is the desk's own scoring, not the model's opinion:
+      // `recommendSeverity` reads "cardiac arrest" and `recommendType` reads
+      // "cardiac". A model that paraphrased the emergency away would show up
+      // here rather than in a sentence nobody checks.
+      expect(logged?.result).toMatch(/"recommendedSeverity":"critical"/);
+      expect(logged?.result).toMatch(/"recommendedType":"medical"/);
+      // `incident_create` is ungated and SPREADS the position it landed in, so
+      // the model reads "confirm the severity and type" as part of this result.
+      expect(logged?.result).toMatch(/"state":"working\.triaging"/);
+      // And the board the browser holds has it, at the severity the desk chose.
+      expect(boardEntry(session.events(), FIRST_INCIDENT)).toMatchObject({
+        severity: "critical",
+        location: "400 Oak Street",
+      });
+    },
+    { stubReply: [...LOG_TURN] },
+  );
+
+  test(
+    "rolls units on a logged incident and follows them to monitoring",
+    async ({ session }) => {
+      const turns = await sayAll(session, [THE_CALL, "Dispatch the recommended units now."]);
+
+      const dispatching = turns.find((t) =>
+        t.toolCalls.some((c) => c.name === "resources_dispatch"),
+      );
+      const rolled = dispatching?.toolCalls.find((c) => c.name === "resources_dispatch");
+      // Units really assigned — `dispatched` is empty when every requested
+      // callsign was busy, which is the case the fourth test owns.
+      expect(rolled?.result).toMatch(/"dispatched":\[\{/);
+      // `sendFrom` only fires when something rolled, so this is the position
+      // moving BECAUSE of the dispatch rather than alongside it.
+      expect(rolled?.result).toMatch(/"state":"working\.monitoring"/);
+      // The board agrees: the incident is dispatched, not merely triaged.
+      expect(boardEntry(session.events(), FIRST_INCIDENT)?.status).toBe("dispatched");
+      // And the order is the one the desk's flow requires: log, then dispatch.
+      const names = session.toolCalls().map((c) => c.name);
+      expect(names.indexOf("resources_dispatch")).toBeGreaterThan(names.indexOf("incident_create"));
+    },
+    {
+      stubReply: [
+        ...LOG_TURN,
+        { tool: "resources_dispatch", args: { incidentId: FIRST_INCIDENT, autoDispatch: true } },
+        "Medic-1 is rolling priority one, ETA under five.",
+      ],
+    },
+  );
+
+  test(
+    "closing an incident releases the units that were on it",
+    async ({ session }) => {
+      await sayAll(session, [
+        THE_CALL,
+        "Dispatch the recommended units, emergency priority.",
+        "Units report the patient is transported and they're clear. Close it out.",
+      ]);
+
+      const closed = callsTo(session, "incident_update_status").filter(
+        (c) => c.args.status === "resolved",
+      );
+      expect(closed).toHaveLength(1);
+      // The release is the DESK's, not the model's: `resolved` is the one status
+      // that detaches every unit still assigned to this incident and says so on
+      // the incident's own timeline. A model cannot route around it, which is
+      // what makes this the claim worth asserting live. (The BUSY-unit refusal
+      // an earlier draft tried to assert here is the case below: a competent
+      // dispatcher checks availability first and never triggers it, so it is
+      // `{ scripted: true }` rather than weakened into this one.)
+      expect(closed[0]?.result).toMatch(/"newStatus":"resolved"/);
+      expect(closed[0]?.result).toMatch(/All resources released/);
+      // The board agrees, which is the half a browser would show.
+      expect(boardEntry(session.events(), FIRST_INCIDENT)?.status).toBe("resolved");
+      // And the shift ran in the order the flow requires.
+      const names = session.toolCalls().map((c) => c.name);
+      expect(names.indexOf("incident_create")).toBeGreaterThanOrEqual(0);
+      expect(names.indexOf("resources_dispatch")).toBeGreaterThan(names.indexOf("incident_create"));
+      expect(names.lastIndexOf("incident_update_status")).toBeGreaterThan(
+        names.indexOf("resources_dispatch"),
+      );
+    },
+    {
+      stubReply: [
+        ...LOG_TURN,
+        {
+          tool: "resources_dispatch",
+          args: { incidentId: FIRST_INCIDENT, autoDispatch: true, priority: "emergency" },
+        },
+        "Medic-1 rolling priority one to 400 Oak Street.",
+        {
+          tool: "incident_update_status",
+          args: { incidentId: FIRST_INCIDENT, status: "resolved", notes: "Patient transported" },
+        },
+        "Copy — Oak Street is closed and the units are clear.",
+      ],
+    },
+  );
+
+  test(
+    "a unit already rolling is not sent to a second call",
+    async ({ session }) => {
+      await sayAll(session, [
+        THE_CALL,
+        "Send Medic-1 to Oak Street.",
+        "New call: 12 Pine Lane, chest pains. Log it.",
+        "Send Medic-1 to Pine Lane as well.",
+      ]);
+
+      const dispatches = callsTo(session, "resources_dispatch");
+      // The second request is the subject, so both have to have gone out.
+      expect(dispatches).toHaveLength(2);
+      const [first, second] = dispatches;
+      // Medic-1 really rolled the first time — otherwise the refusal below is
+      // about a unit that was never busy.
+      expect(first?.result).toMatch(/"callsign":"Medic-1"/);
+      expect(first?.result).toMatch(/"state":"working\.monitoring"/);
+      // And the second time the desk REFUSED rather than double-booking it:
+      // `failed` carries the reason and `dispatched` is empty. The unit is
+      // committed to ONE incident, which is the property a dispatch desk is
+      // useless without and which no prompt can carry.
+      expect(second?.result).toMatch(/"dispatched":\[\]/);
+      expect(second?.result).toMatch(/"callsign":"Medic-1","reason":"Currently dispatched"/);
+      // `sendFrom` only fires when something rolled, so the call did NOT
+      // advance: logging Pine Lane put it back at `triaging`, and a dispatch
+      // that dispatched nothing leaves it there.
+      expect(second?.result).toMatch(/"state":"working\.triaging"/);
+      // The board agrees on both halves — the second incident never reached
+      // `dispatched`, and the first one still holds the unit.
+      expect(boardEntry(session.events(), SECOND_INCIDENT)?.status).not.toBe("dispatched");
+      expect(boardEntry(session.events(), FIRST_INCIDENT)?.status).toBe("dispatched");
+    },
+    // Scripted only. `resources_get_available` is in this desk's tool list and
+    // the prompt tells it to check capacity, so a live model looks first and
+    // never asks for a busy unit — which is the right behaviour and leaves the
+    // refusal unobserved. Forcing the call is the only way to see the guard
+    // itself, and a case that "accepts either" would assert nothing live.
+    {
+      scripted: true,
+      stubReply: [
+        ...LOG_TURN,
+        {
+          tool: "resources_dispatch",
+          args: { incidentId: FIRST_INCIDENT, callsigns: ["Medic-1"], priority: "emergency" },
+        },
+        "Medic-1 is rolling to 400 Oak Street.",
+        {
+          tool: "incident_create",
+          args: { location: "12 Pine Lane", description: "Chest pains" },
+        },
+        "Logged as a second incident.",
+        {
+          tool: "resources_dispatch",
+          args: { incidentId: SECOND_INCIDENT, callsigns: ["Medic-1"], priority: "priority" },
+        },
+        "Medic-1 is already committed to Oak Street — I'll find you another unit.",
+      ],
+    },
+  );
+});

package/dist/templates/embedded-assets/agent.eval.test.ts

@@ -0,0 +1,138 @@
+/// <reference types="vite/client" />
+
+// An EVAL: does the FAQ bot actually look things up?
+//
+// `agent.test.ts` scores the search index directly, which settles what
+// `search_knowledge` returns for a query it is handed. What it cannot settle is
+// the discipline this agent's whole prompt is about: that an answer comes from
+// `knowledge.json` and not from what a model happens to know about the
+// framework it is describing.
+//
+// Run it with `aai eval`. Without a provider key each case runs against a
+// SCRIPTED model (its `stubReply`) — the real session and the real tools, a
+// fake reply — which proves the wiring and says nothing about the discipline.
+
+import { withSystemPrompt } from "@alexkroman1/aai/manifest";
+import { withDiscoveredTools } from "@alexkroman1/aai/testing";
+import { toolResultIn } from "@alexkroman1/aai-runtime/eval";
+import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import { z } from "zod";
+import authoredAgent from "./agent.ts";
+import { faqs } from "./shared.ts";
+import systemPrompt from "./system-prompt.md?raw";
+
+/**
+ * The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
+ *
+ * The glob is written HERE rather than reached for from a shared helper because
+ * this file SHIPS — a scaffolded project has no repo helper to import. Without
+ * it this agent would have no tools and would answer every question from the
+ * model's own memory, which is exactly what the cases below forbid.
+ *
+ * And plus its PROMPT. `agent.ts` does not declare one, because
+ * `system-prompt.md` is resolved by the BUILD (`aai build`/`aai deploy`) — so
+ * the raw default export carries the FRAMEWORK DEFAULT prompt. An eval that
+ * drives it measures a different agent than the one that deploys, and every
+ * tool-choice claim below then passes or fails for the wrong reason.
+ */
+const agentDef = withSystemPrompt(
+  withDiscoveredTools(authoredAgent, import.meta.glob("./tools/*.ts", { eager: true })),
+  systemPrompt,
+);
+
+/**
+ * A knowledge-base HIT, as the model saw it — `tool.completed` carries it
+ * serialized, and `toolResultIn` parses and validates it.
+ *
+ * A schema rather than a cast, which is the whole reason to pass one: an index
+ * that started answering with a different shape FAILS here naming the field,
+ * where the cast this replaced would have read `undefined` off it and failed a
+ * line later on something unrelated. `search_knowledge` also answers
+ * `{ result: "No matching FAQ found." }` for a miss, so a miss fails HERE — and
+ * a miss on this question is the finding.
+ */
+const FaqEntrySchema = z.object({ question: z.string(), answer: z.string() });
+
+/** The knowledge-base entry this template's own answer about the web is in. */
+const INTERNET = faqs.find((f) => f.question.includes("internet"))!;
+
+describeEval(agentDef, (test) => {
+  test(
+    "answers a PARAPHRASED question out of the knowledge base",
+    async ({ session }) => {
+      // Nothing in this wording appears in the entry's question, so the entry
+      // has to be found by the index rather than matched by substring — and the
+      // agent has to go looking instead of telling the caller what it knows
+      // about voice frameworks in general.
+      const turn = await session.say("Can your agents make HTTP requests?");
+
+      expect(turn.toolCalls.map((c) => c.name)).toEqual(["search_knowledge"]);
+      const call = turn.toolCalls[0]!;
+      expect(typeof (call.args as { query?: unknown }).query).toBe("string");
+      // The right entry, out of four: the one this question is really about.
+      expect(toolResultIn(turn.toolCalls, "search_knowledge", FaqEntrySchema).question).toBe(
+        INTERNET.question,
+      );
+      // And the reply is that entry's answer rather than an embellishment of
+      // it — "quote the knowledge base accurately" is the prompt's rule.
+      expect(turn.text).toMatch(/fetch|web_search|http/i);
+    },
+    {
+      stubReply: [
+        { tool: "search_knowledge", args: { query: "Can your agents make HTTP requests?" } },
+        "Yes — agents run with network access and can use the fetch API or the web_search builtin.",
+      ],
+    },
+  );
+
+  test(
+    "lists the topics it really has, not the ones it can imagine",
+    async ({ session }) => {
+      const turn = await session.say("What topics can you help me with?");
+
+      expect(turn.toolCalls.map((c) => c.name)).toEqual(["list_topics"]);
+      // Every question in `knowledge.json` and nothing else — the check that
+      // catches an index built from a stale copy of the asset.
+      expect(toolResultIn(turn.toolCalls, "list_topics", z.array(z.string()))).toEqual(
+        faqs.map((f) => f.question),
+      );
+      expect(turn.completed).toBe(true);
+    },
+    {
+      stubReply: [
+        { tool: "list_topics" },
+        "I can cover what AAI is, how tools work, speech providers, and network access.",
+      ],
+    },
+  );
+
+  test(
+    "says it does not know rather than answering off-base",
+    async ({ session }) => {
+      // The knowledge base has four entries and none of them is the weather.
+      // A model asked this will answer it unless the prompt holds — and an FAQ
+      // bot that answers from outside its own asset is the failure this
+      // template exists to demonstrate the fix for.
+      const turn = await session.say("What's the weather in Paris right now?");
+
+      expect(turn.text).toMatch(/can.?t|cannot|do(n.?t| not) have|not something I/i);
+      // No degrees, no forecast: whatever it says, it must not have invented an
+      // answer, and it may not have found one in a knowledge base without one.
+      expect(turn.text).not.toMatch(/\d+\s*(°|degrees)/i);
+      for (const call of turn.toolCalls) {
+        // `toolResultIn` over a ONE-CALL list: the name is this call's own, so
+        // the reader's "no such call" and "two calls" throws are unreachable and
+        // what is left is the parse plus its "never completed" failure — which
+        // is exactly what the local helper this replaced did by hand.
+        expect(toolResultIn([call], call.name)).not.toMatchObject({
+          question: expect.stringContaining("weather"),
+        });
+      }
+    },
+    // Live only: a scripted model saying "I don't know" proves that the script
+    // said so, which is not the claim. `{ live: true }` is the honest way to
+    // write a case a stub cannot satisfy.
+    { live: true },
+  );
+});

package/dist/templates/health-assistant/agent.eval.test.ts

@@ -0,0 +1,167 @@
+/// <reference types="vite/client" />
+
+// An EVAL: does the desk read the label, or does it remember?
+//
+// `agent.test.ts` drives both tools against a faked openFDA, which settles the
+// cross-mention scan and the refuse-on-a-missing-drug rule. What it cannot
+// settle is the one thing this agent is dangerous without: that a question
+// about two medications reaches `check_drug_interaction` at all, with BOTH
+// drugs in it, instead of being answered from what a model has read about
+// pharmacology.
+//
+// Run it with `aai eval`. Without a provider key each case runs against a
+// SCRIPTED model (its `stubReply`) — and a scripted tool call really executes,
+// so a keyless run still covers this template's own code, including the
+// refusal. It says nothing about what the agent CHOSE.
+//
+// Note what these cases deliberately do NOT assert: anything about a label's
+// contents. The tools really call openFDA, so an assertion about what a label
+// says is an assertion about a third party's uptime — while "which tool, with
+// which arguments" is a fact about the agent and holds either way (an
+// unreachable openFDA reads as a drug that could not be resolved, which this
+// template already refuses on).
+
+import { withSystemPrompt } from "@alexkroman1/aai/manifest";
+import { withDiscoveredTools } from "@alexkroman1/aai/testing";
+import { toolResultIn } from "@alexkroman1/aai-runtime/eval";
+import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import { z } from "zod";
+import authoredAgent from "./agent.ts";
+import systemPrompt from "./system-prompt.md?raw";
+
+/**
+ * The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
+ *
+ * The glob is written HERE rather than reached for from a shared helper because
+ * this file SHIPS — a scaffolded project has no repo helper to import. Without
+ * it the desk would have no tools and would answer every one of these from
+ * memory, which is the failure these cases exist to catch.
+ *
+ * And plus its PROMPT. `agent.ts` does not declare one, because
+ * `system-prompt.md` is resolved by the BUILD (`aai build`/`aai deploy`) — so
+ * the raw default export carries the FRAMEWORK DEFAULT prompt. An eval that
+ * drives it measures a different agent than the one that deploys, and every
+ * tool-choice claim below then passes or fails for the wrong reason.
+ */
+const agentDef = withSystemPrompt(
+  withDiscoveredTools(authoredAgent, import.meta.glob("./tools/*.ts", { eager: true })),
+  systemPrompt,
+);
+
+/** The drugs an interaction check was actually asked about, lowercased. */
+const drugsIn = (args: Record<string, unknown>): string[] =>
+  ((args.drugs as string[] | undefined) ?? []).map((d) => d.trim().toLowerCase());
+
+/**
+ * A refusal from `check_drug_interaction`, as the model saw it.
+ *
+ * A schema rather than a cast, which is what `toolResultIn` takes one for: a
+ * result that stopped carrying `error` fails HERE naming the field, where the
+ * cast this replaced read `undefined` off it and the assertion below then said
+ * "expected undefined to match /sparkleforin/" without saying why.
+ */
+const Refusal = z.object({ error: z.string() });
+
+describeEval(agentDef, (test) => {
+  test(
+    "checks the labels for an interaction, and still sends the caller to a human",
+    async ({ session }) => {
+      const turn = await session.say("Can I take ibuprofen and warfarin together?");
+
+      // It may look each drug up as well — that is fine and often useful. What
+      // it may not do is answer this question without asking the labels.
+      const checks = turn.toolCalls.filter((c) => c.name === "check_drug_interaction");
+      expect(checks).not.toEqual([]);
+      const asked = checks.flatMap((c) => drugsIn(c.args));
+      expect(asked).toContain("ibuprofen");
+      expect(asked).toContain("warfarin");
+
+      // The rule at the top of its prompt: it is not a doctor. An interaction
+      // answer that does not end at a professional is the failure that makes
+      // this whole template a liability.
+      expect(turn.text).toMatch(/doctor|pharmacist|provider|healthcare|professional/i);
+    },
+    {
+      stubReply: [
+        { tool: "check_drug_interaction", args: { drugs: ["ibuprofen", "warfarin"] } },
+        "The labels do mention each other — please confirm with your doctor or pharmacist.",
+      ],
+    },
+  );
+
+  test(
+    "checks the drug the caller mentioned a turn ago, not just the new one",
+    async ({ session }) => {
+      await session.say("I take warfarin every morning.");
+      // Only ibuprofen is named here. A check that goes out with one drug in it
+      // is the dangerous shape: `check_drug_interaction` needs two, so a desk
+      // that forgot the first will either refuse or — worse — look up the new
+      // drug alone and report nothing.
+      const turn = await session.say("Is it okay if I add ibuprofen for a headache?");
+
+      const checks = turn.toolCalls.filter((c) => c.name === "check_drug_interaction");
+      expect(checks).not.toEqual([]);
+      const asked = checks.flatMap((c) => drugsIn(c.args));
+      expect(asked).toContain("warfarin");
+      expect(asked).toContain("ibuprofen");
+    },
+    {
+      stubReply: [
+        "Good to know — warfarin it is.",
+        { tool: "check_drug_interaction", args: { drugs: ["warfarin", "ibuprofen"] } },
+        "Their labels mention each other, so check with your doctor before adding it.",
+      ],
+    },
+  );
+
+  test(
+    "does not promise a lookup it never makes",
+    async ({ session }) => {
+      // This one has been SEEN to fail, and what it caught is worth knowing:
+      // driven against the FRAMEWORK DEFAULT prompt — i.e. with this template's
+      // `system-prompt.md` not applied, which is what an eval on the raw default
+      // export measures — the same utterance got "I'll look up details about
+      // warfarin for you." and made no tool call at all, so the caller was told
+      // a lookup was happening and given nothing. Under this agent's own prompt
+      // it passes. A promise is only a promise if the turn it is in keeps it,
+      // and the prompt is what makes that true.
+      const turn = await session.say("I take warfarin every morning.");
+
+      if (turn.toolCalls.length === 0) {
+        expect(turn.text).not.toMatch(
+          /(I'?|I wi)ll (look|check|pull|find)|let me (look|check|pull)/i,
+        );
+      }
+      expect(turn.completed).toBe(true);
+    },
+    { stubReply: "Noted — warfarin every morning. What would you like to know about it?" },
+  );
+
+  test(
+    "a drug it cannot find is a refusal, never a clean bill of health",
+    async ({ session }) => {
+      const turn = await session.say("Is it safe to take sparkleforin with aspirin?");
+
+      // Whatever it does, it may not tell the caller this combination is fine:
+      // nothing here could have established that.
+      expect(turn.text).not.toMatch(/no (known )?interaction|safe to (take|combine|mix)/i);
+      // And any check it did run had to REFUSE rather than report zero
+      // interactions — the rule that stops an unresolvable drug being silently
+      // dropped from the comparison.
+      for (const call of turn.toolCalls.filter((c) => c.name === "check_drug_interaction")) {
+        // `toolResultIn` over a ONE-CALL list: the name is this call's own, so
+        // the reader's "no such call" and "two calls" throws are unreachable,
+        // and what is left is the parse, the schema, and the "never completed"
+        // failure the local helper used to hand-roll.
+        expect(toolResultIn([call], call.name, Refusal).error).toMatch(/sparkleforin/i);
+      }
+    },
+    {
+      stubReply: [
+        { tool: "check_drug_interaction", args: { drugs: ["sparkleforin", "aspirin"] } },
+        "I could not find a label for sparkleforin — could you check the spelling?",
+      ],
+    },
+  );
+});