@alexkroman1/aai-cli 13.1.0 → 13.3.0

package/dist/templates/research-workflow/agent.eval.test.ts

@@ -29,8 +29,9 @@
 // endpointing, barge-in, whether two sentences merged into one turn.
 
 import agentDef from "virtual:aai/agent";
+import { stubGatewayRoute } from "@alexkroman1/aai/testing";
 import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
-import type { EvalToolCall, EvalWorkflows } from "@alexkroman1/aai-runtime/eval";
+import { type EvalToolCall, type EvalWorkflows, toolResultIn } from "@alexkroman1/aai-runtime/eval";
 import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
 import { expect } from "vitest";
 import { z } from "zod";
@@ -112,7 +113,17 @@ type ScriptedSteps = {
  * on the live path, so a live case still measures the agent. Anything that is
  * not the gateway THROWS rather than answering 200: an unexpected request from
  * a step is a finding, and a silent empty body would be read as a model that
- * said nothing.
+ * said nothing. `stubGatewayRoute` is what decides which is which, and it
+ * decides on the SDK's own completions PATH — so the script cannot come unstuck
+ * from the step by the two agreeing on a typo, and the envelope it answers with
+ * is the SDK's rather than this file's. That last part is the one worth having:
+ * the envelope is a WIRE shape, so a field typed one off does not fail —
+ * `stepGenerate` reads no content, reports an empty completion, and the case
+ * blames the run.
+ *
+ * The CURSOR is the reader's too: the last reply repeats, which is what a stage
+ * that legitimately calls the model twice needs and what stops a script running
+ * out mid-run and failing on itself.
  *
  * `hold` keeps the FIRST answer pending, which is the only way to observe a run
  * that is still going: a durable `sleep` is skipped here, so an unheld run
@@ -120,15 +131,17 @@ type ScriptedSteps = {
  */
 function scriptSteps(options: { hold?: boolean } = {}): ScriptedSteps {
   const gate = Promise.withResolvers<void>();
-  let served = 0;
+  const model = stubGatewayRoute(MODEL_SCRIPT);
   const stub = installStubStepFetch(async (request) => {
-    if (!request.url.includes("/chat/completions")) {
+    const answered = model.route(request);
+    if (answered === undefined) {
       throw new Error(`unexpected step request in an eval: ${request.method} ${request.url}`);
     }
-    const at = served++;
-    if (options.hold === true && at === 0) await gate.promise;
-    const content = MODEL_SCRIPT[Math.min(at, MODEL_SCRIPT.length - 1)];
-    return { body: { choices: [{ message: { content } }] } };
+    // `model.calls` has already recorded this one, so a length of 1 IS the first
+    // answer — and holding after the route rather than before it keeps the reply
+    // this returns the one the script owed that call.
+    if (options.hold === true && model.calls.length === 1) await gate.promise;
+    return answered;
   });
   return { calls: stub.calls, release: () => gate.resolve() };
 }
@@ -141,20 +154,16 @@ const Started = z.object({
 });
 
 /**
- * The run id a `request_research` call reported.
+ * The run id the `request_research` call reported.
  *
- * Parsed rather than regexed: a tool result reaches the event stream as a
- * serialized string, and a shape that stopped matching should fail HERE naming
- * the field instead of handing the next assertion `undefined`.
+ * `toolResultIn` rather than a `find` and a parse: a tool result reaches the
+ * event stream as a serialized string, and a shape that stopped matching should
+ * fail HERE naming the field instead of handing the next assertion `undefined`.
+ * It throws for the two other ways this can go wrong as well, each naming what
+ * was really called — no such call, and a call that never returned.
  */
 function startedRunId(calls: readonly EvalToolCall[]): string {
-  const call = calls.find((one) => one.name === "request_research");
-  if (call?.result === undefined) {
-    throw new Error(
-      `the desk called no request_research: ${calls.map((one) => one.name).join(", ") || "(no tools)"}`,
-    );
-  }
-  return Started.parse(JSON.parse(call.result)).runId;
+  return toolResultIn(calls, "request_research", Started).runId;
 }
 
 /** Every tool call in this turn that READS a run, whichever the model picked. */
@@ -177,12 +186,13 @@ const START_TURN = [
  * Not tidiness: the scripted `stepFetch` is unpublished when the test that
  * installed it finishes, so a body still mid-flight would make its next model
  * call against whatever the next case publishes — or against the real gateway.
+ * `close()` reports that on stderr (`EvalRunAbandoned`) rather than fixing it:
+ * the wait is `settleAll`'s, and the RELEASE stays here, because what holds the
+ * run in flight is this file's own gate and nothing in the harness can open one.
  */
 async function drain(workflows: EvalWorkflows | undefined, steps: ScriptedSteps): Promise<void> {
   steps.release();
-  for (const run of await (workflows?.runs() ?? [])) {
-    await workflows?.settle(run.runId, research);
-  }
+  await workflows?.settleAll();
 }
 
 describeEval(

package/dist/templates/research-workflow/agent.test.ts

@@ -106,7 +106,13 @@ function stubWorkflows(runs: WorkflowRunSnapshot[] = []): WorkflowClient {
 
 describe("the agent declares its workflow", () => {
   test("under the name ctx.workflows.start resolves it by", () => {
-    expect(Object.keys(agentDef.workflows ?? {})).toEqual(["research"]);
+    // `toContain` rather than an exact key list: a second workflow is an
+    // invited edit and must not redden a test the author did not write. The
+    // NAME is still pinned, deliberately — this key is a STRING to everything
+    // outside this file (the REST route, `ctx.workflows.get`, a schedule), so
+    // renaming it is a runtime 404 rather than a compile error, and nothing
+    // else says so.
+    expect(Object.keys(agentDef.workflows ?? {})).toContain("research");
     expect(agentDef.workflows?.research).toBe(research);
   });
 

package/dist/templates/retail/agent.eval.test.ts

@@ -31,7 +31,7 @@ import type { SessionEvent } from "@alexkroman1/aai/protocol";
 // What no eval here can see: anything below the audio boundary. Whether a
 // caller reading an order number in bursts lands as one turn is a property of
 // endpointing, and these fake speech stages remove it.
-import { type EvalSession, type EvalTurn, lastStateIn } from "@alexkroman1/aai-runtime/eval";
+import { describeTurn, lastStateIn, toolNames, turnCalling } from "@alexkroman1/aai-runtime/eval";
 import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
 import { expect } from "vitest";
 import { z } from "zod";
@@ -93,26 +93,6 @@ function statusOf(events: readonly SessionEvent[], orderId: string): string | un
 const refusalAt = (state: string) =>
   new RegExp(`Not available yet: this conversation is at [\\\\"]*${state}`);
 
-/**
- * Drive a whole call, one caller line at a time, and hand back every turn.
- *
- * The cases below assert about the turn a MECHANISM fired in rather than about
- * turn number two, because how many turns a desk takes to get there is the
- * model's business and it really does vary: measured live, this agent reads the
- * order back out of `get_order_details` and asks before it stages, so the
- * staging call has landed in turn two, three and four across runs. A case
- * pinned to a turn index is a flake with a misleading name.
- */
-async function sayAll(session: EvalSession, lines: readonly string[]): Promise<EvalTurn[]> {
-  const turns: EvalTurn[] = [];
-  for (const line of lines) turns.push(await session.say(line));
-  return turns;
-}
-
-/** The turn a named tool was called in, if any. */
-const turnCalling = (turns: readonly EvalTurn[], tool: string) =>
-  turns.find((t) => t.toolCalls.some((c) => c.name === tool));
-
 /** One line the caller says to identify themselves, and the scripted tool call
  *  that answers it — the first turn of three of these four cases. */
 const AUTH_TURN = [
@@ -175,7 +155,7 @@ describeEval(retailAgent, (test) => {
       // turn, and whether it spends one is not something an eval should pin.
       // Every assertion below is about the turn the staging landed in, so a
       // later apply cannot affect any of them.
-      const turns = await sayAll(session, [
+      const turns = await session.sayAll([
         `My email is ${CALLER_EMAIL}.`,
         "I'd like to cancel my pending order — I ordered it by mistake.",
         "Yes, please go ahead and cancel it.",
@@ -183,8 +163,13 @@ describeEval(retailAgent, (test) => {
         "Yes. Cancel it, please.",
       ]);
 
+      // The turn the staging landed in, whichever it was — measured live it has
+      // been turn two, three and four. `turnCalling` throws when no turn staged
+      // at all, naming every turn's tool list: a desk that talked through all
+      // five without staging is the finding, and "expected undefined to be
+      // defined" is not a report of it.
       const staging = turnCalling(turns, "cancel_pending_order");
-      const staged = staging?.toolCalls.find((c) => c.name === "cancel_pending_order");
+      const staged = staging.toolCalls.find((c) => c.name === "cancel_pending_order");
       expect(staged?.result).toMatch(/NOTHING HAS CHANGED YET/);
       // The gate is a POSITION, and this is it moving: the tool reported the
       // state it landed in, which is the only state `confirm_change` is legal
@@ -192,13 +177,13 @@ describeEval(retailAgent, (test) => {
       expect(staged?.result).toMatch(/serving\.awaitingConfirmation/);
       // A change cannot be described and applied in the same turn. This is the
       // property the prose in the system prompt could never have.
-      expect(staging?.toolCalls.map((c) => c.name)).not.toContain("confirm_change");
+      expect(toolNames(staging.toolCalls)).not.toContain("confirm_change");
       // And after the turn that staged it, the store really is untouched — read
       // off the projection the BROWSER was sent in that same turn.
-      expect(statusOf(staging?.events ?? [], PENDING_ORDER)).toBe("pending");
-      expect(projection(staging?.events ?? [])?.pending?.kind).toBe("cancel_pending_order");
+      expect(statusOf(staging.events, PENDING_ORDER)).toBe("pending");
+      expect(projection(staging.events)?.pending?.kind).toBe("cancel_pending_order");
       // Step 2 of the policy: read it back and ask.
-      expect(staging?.text).toMatch(/\?/);
+      expect(staging.text).toMatch(/\?/);
     },
     { stubReply: [...AUTH_TURN, ...STAGE_TURN, "Cancelling it now — one moment."] },
   );
@@ -210,7 +195,7 @@ describeEval(retailAgent, (test) => {
       // in is its own business, and saying yes repeatedly is what makes
       // "exactly once" below a claim about the MECHANISM rather than about the
       // model's pacing.
-      await sayAll(session, [
+      await session.sayAll([
         `My email is ${CALLER_EMAIL}.`,
         "Please cancel my pending order — I ordered it by mistake.",
         "Yes, that's right, go ahead.",
@@ -231,7 +216,7 @@ describeEval(retailAgent, (test) => {
         expect(extra.result).toMatch(/Not available yet/);
       }
       // And it came after the stage, never instead of it.
-      const names = session.toolCalls().map((c) => c.name);
+      const names = toolNames(session.toolCalls());
       expect(names.indexOf("cancel_pending_order")).toBeGreaterThanOrEqual(0);
       expect(names.indexOf("confirm_change")).toBeGreaterThan(
         names.indexOf("cancel_pending_order"),
@@ -262,11 +247,10 @@ describeEval(retailAgent, (test) => {
       // Named first, and with a message: a live model that answers the request
       // with a question instead of the tool leaves `transfer` undefined, and
       // `.toMatch()` on it reports only "expected a string, got undefined" —
-      // which says nothing about what the desk actually did.
-      expect(
-        transfer,
-        `tools called: [${handoff.toolCalls.map((c) => c.name).join(", ")}]; said: ${handoff.text}`,
-      ).toBeDefined();
+      // which says nothing about what the desk actually did. `describeTurn` is
+      // that sentence, done by the harness: the tools it called and what it
+      // said, plus whether the reply was cancelled.
+      expect(transfer, describeTurn(handoff)).toBeDefined();
       // The terminal state is what makes "say nothing else after this" a
       // property of the agent rather than a line in its prompt: `done` is the
       // flow saying there is nowhere left to go.

package/dist/templates/retail/registry.test.ts

@@ -1,21 +1,6 @@
 import { isToolFailure, type ToolContext } from "@alexkroman1/aai";
 import { createToolContext, deployedAgent } from "@alexkroman1/aai/testing";
 import { describe, expect, test } from "vitest";
-import { callFlow, retailSlot } from "./store.ts";
-
-/** Tools that legitimately run before the caller is identified — the six
- *  declaring `when: BEFORE_TRANSFER`. Everything else must refuse. Listed here
- *  so ADDING an unauthenticated tool is a deliberate edit to this file, not a
- *  silent gap. */
-const PUBLIC_TOOLS = new Set([
-  "find_user_id_by_email",
-  "find_user_id_by_name_zip",
-  "get_product_details",
-  "get_item_details",
-  "list_all_product_types",
-  "transfer_to_human_agents",
-]);
-
 /**
  * The def a DEPLOYED agent runs, lowered BY HAND — the one place in the
  * templates that still does.
@@ -27,6 +12,7 @@ const PUBLIC_TOOLS = new Set([
  * whose runner is not vitest, and so cannot register the plugin.
  */
 import authoredAgent from "./agent.ts";
+import { callFlow, gateFor, retailSlot } from "./store.ts";
 import systemPrompt from "./system-prompt.md?raw";
 
 const retailAgent = deployedAgent(authoredAgent, {
@@ -36,6 +22,39 @@ const retailAgent = deployedAgent(authoredAgent, {
 
 const registry = Object.entries(retailAgent.tools);
 
+/**
+ * Whether `name`'s gate admits the pre-identification state — i.e. whether it
+ * is one of the tools that legitimately runs before the caller is identified.
+ *
+ * ASKED OF THE REGISTRY, not read off a list here. This used to be a hardcoded
+ * six-name set, which meant a tool you add is classified by a file you did not
+ * write: declare `when: BEFORE_TRANSFER` on a new catalogue read and the
+ * sweeps below would have demanded it refuse. `retailTool` records every gate
+ * in `TOOL_GATES`, so the honest question is what the tool itself declared.
+ *
+ * `undefined` — a tool built with plain `tool()` rather than `retailTool` — is
+ * neither: it has no gate at all, and the sweeps skip it. That is the same
+ * answer `gateFor` gives the `tool.called` hook, and "a tool this template did
+ * not declare is ignored" below is the assertion for it.
+ */
+const isPublic = (name: string) => gateFor(name)?.includes("identifying") ?? false;
+
+/** Registry entries that went through `retailTool`, so the gate has an opinion. */
+const gatedTools = registry.filter(([name]) => gateFor(name) !== undefined);
+
+/** The tools this template SHIPS that run before identification. A pin on the
+ *  shipped set — one of these quietly losing `BEFORE_TRANSFER` is a policy
+ *  break — and nothing else reads it, so adding a public tool of your own costs
+ *  no edit here. */
+const SHIPPED_PUBLIC_TOOLS = [
+  "find_user_id_by_email",
+  "find_user_id_by_name_zip",
+  "get_product_details",
+  "get_item_details",
+  "list_all_product_types",
+  "transfer_to_human_agents",
+];
+
 // `createToolContext()` rather than a cast: it carries a real slot store (the
 // same storability check and freeze the deployed one applies), and each call is a
 // distinct session, which is what these per-tool cases assume.
@@ -81,9 +100,19 @@ async function bodyReachableCtx(name: string): Promise<ToolContext> {
   return ctx;
 }
 
-/** Minimal args satisfying each tool's schema. Deliberately plausible-shaped
- *  but wrong — these calls are expected to fail; what is asserted is that they
- *  still moved the UI. */
+/**
+ * Minimal args satisfying each tool's schema. Deliberately plausible-shaped but
+ * wrong — these calls are expected to fail; what is asserted is that they still
+ * moved the UI.
+ *
+ * **It is also the list of tools this file sweeps, and the list of tools this
+ * template ships.** It used to be a second copy of a seventeen-name array
+ * asserted with `toEqual`, so adding one tool of your own failed both — the
+ * count and the coverage — before it had run once. One source now: a tool with
+ * an entry here is driven through every sweep below, and every name here must
+ * still be discovered from `tools/`. Add your tool with its arguments and it
+ * joins the sweeps; leave it out and it is simply not covered by them.
+ */
 const SAMPLE_ARGS: Record<string, Record<string, unknown>> = {
   cancel_change: {},
   cancel_pending_order: { order_id: "#W0000000", reason: "no longer needed" },
@@ -137,56 +166,56 @@ const SAMPLE_ARGS: Record<string, Record<string, unknown>> = {
   transfer_to_human_agents: { summary: "test" },
 };
 
+/** The tools with sample args, i.e. the ones every sweep below can drive. */
+const sweepable = registry.filter(([name]) => name in SAMPLE_ARGS);
+
 describe("tool registry", () => {
-  test("registers all seventeen tools", () => {
-    // Fifteen of these are tau2's retail tool set, which this template used to
-    // hold verbatim. `confirm_change` and `cancel_change` are the two it does
-    // not have: tau2's tools apply on their first call, and here nothing does —
-    // see `pending.ts`. Departing from that set is what buys the confirmation
-    // gate, and it is the reason this list is no longer a fidelity claim.
-    expect(registry.map(([name]) => name).sort()).toEqual(
-      [
-        "cancel_change",
-        "cancel_pending_order",
-        "confirm_change",
-        "exchange_delivered_order_items",
-        "find_user_id_by_email",
-        "find_user_id_by_name_zip",
-        "get_item_details",
-        "get_order_details",
-        "get_product_details",
-        "get_user_details",
-        "list_all_product_types",
-        "modify_pending_order_address",
-        "modify_pending_order_items",
-        "modify_pending_order_payment",
-        "modify_user_address",
-        "return_delivered_order_items",
-        "transfer_to_human_agents",
-      ].sort(),
+  test("discovers every tool this template ships", () => {
+    // `arrayContaining` rather than an exact sorted list. Fifteen of these are
+    // tau2's retail tool set, which this template used to hold verbatim;
+    // `confirm_change` and `cancel_change` are the two it does not have, since
+    // tau2's tools apply on their first call and here nothing does (see
+    // `pending.ts`). So the list is not a fidelity claim and never was a
+    // COUNT — what it is worth asserting is that discovery still finds each
+    // one, because a `tools/` glob that resolves nothing looks exactly like a
+    // desk with no tools. Adding a tool of your own passes; losing one of these
+    // fails, naming it.
+    expect(registry.map(([name]) => name)).toEqual(
+      expect.arrayContaining(Object.keys(SAMPLE_ARGS)),
     );
   });
 
-  // Both sweep the registry, so they assert softly: adding a batch of tools
-  // should list every one still missing its entry, not just the first.
-  test("every registered tool has sample args, so the sweeps below cover it", () => {
-    for (const [name] of registry) {
-      expect.soft(SAMPLE_ARGS[name], `add SAMPLE_ARGS["${name}"]`).toBeDefined();
+  // Asserts softly: a batch of renames should list every name that no longer
+  // resolves, not just the first.
+  test("every name in SAMPLE_ARGS resolves to a real tool", () => {
+    for (const name of Object.keys(SAMPLE_ARGS)) {
+      expect.soft(retailAgent.tools[name], `SAMPLE_ARGS["${name}"] names no tool`).toBeDefined();
     }
   });
 
   test("every tool declares a description the model can act on", () => {
     for (const [name, def] of registry) {
+      // Any tool, yours included — the model picks a tool by its description,
+      // so an empty one is invisible to it.
       expect.soft(def.description, name).toBeTruthy();
-      expect.soft(def.description.length, name).toBeGreaterThan(40);
+    }
+    for (const name of Object.keys(SAMPLE_ARGS)) {
+      // The shipped ones additionally carry enough of the policy to be chosen
+      // correctly, which for this desk means more than a label.
+      expect.soft(retailAgent.tools[name]?.description.length, name).toBeGreaterThan(40);
     }
   });
 });
 
 describe("the UI-update invariant", () => {
-  // This is the one that fails if a future tool is built with tool() instead of
+  // Over `sweepable` rather than the whole registry: these two REACH a tool's
+  // body, so they need arguments its schema accepts, and calling a tool of
+  // yours with `{}` would report a crash inside it as a UI regression. Give it
+  // a `SAMPLE_ARGS` entry and it is swept like the rest.
+  //
+  // This is the pair that fails if a future tool is built with tool() instead of
   // retailTool(): it would work, and the sidebar would sit still through it.
-  test.each(registry)("%s increments callSeq and logs activity", async (name, def) => {
+  test.each(sweepable)("%s increments callSeq and logs activity", async (name, def) => {
     // In whichever state lets the body run, so the flow gate is not what these
     // calls are testing: the point is that a tool which reaches its BODY moves
     // the sidebar. A refused call never reaches one — the hook below is what
@@ -200,7 +229,7 @@ describe("the UI-update invariant", () => {
     expect(state.activity.at(-1)?.summary).toBeTruthy();
   });
 
-  test.each(registry)("%s logs its own registry key as its name", async (name, def) => {
+  test.each(sweepable)("%s logs its own registry key as its name", async (name, def) => {
     // Catches a copy-paste where the retailTool `name` and the registry key
     // disagree — the activity feed would then attribute calls to the wrong tool.
     const ctx = await bodyReachableCtx(name);
@@ -229,7 +258,7 @@ describe("the blocked-call hook", () => {
       ctx,
     );
 
-  test.each(registry.filter(([name]) => !PUBLIC_TOOLS.has(name)))(
+  test.each(gatedTools.filter(([name]) => !isPublic(name)))(
     "%s records a blocked line when the model tries it too early",
     (name) => {
       const ctx = makeCtx();
@@ -245,7 +274,7 @@ describe("the blocked-call hook", () => {
     },
   );
 
-  test.each(registry.filter(([name]) => PUBLIC_TOOLS.has(name)))(
+  test.each(gatedTools.filter(([name]) => isPublic(name)))(
     "%s is left to the wrapper, because it is going to run",
     (name) => {
       const ctx = makeCtx();
@@ -280,7 +309,19 @@ describe("the blocked-call hook", () => {
 });
 
 describe("the authentication gate", () => {
-  test.each(registry.filter(([name]) => !PUBLIC_TOOLS.has(name)))(
+  test("the tools that run before identification are still the shipped six", () => {
+    // The one place `SHIPPED_PUBLIC_TOOLS` is read, and the only assertion in
+    // this file about WHICH tools are public. The sweeps below classify by
+    // `isPublic`, i.e. by what each tool declared, so a public tool of your own
+    // needs no edit here — but one of these six quietly losing
+    // `when: BEFORE_TRANSFER` would silently make the desk unreachable before a
+    // caller is identified, and nothing else would say so.
+    expect(registry.map(([name]) => name).filter(isPublic)).toEqual(
+      expect.arrayContaining(SHIPPED_PUBLIC_TOOLS),
+    );
+  });
+
+  test.each(gatedTools.filter(([name]) => !isPublic(name)))(
     "%s refuses before the caller is identified",
     async (name, def) => {
       const result = await def.execute(SAMPLE_ARGS[name] ?? {}, makeCtx());
@@ -297,7 +338,7 @@ describe("the authentication gate", () => {
     },
   );
 
-  test.each(registry.filter(([name]) => PUBLIC_TOOLS.has(name)))(
+  test.each(sweepable.filter(([name]) => isPublic(name)))(
     "%s does not require authentication",
     async (name, def) => {
       const result = await def.execute(SAMPLE_ARGS[name] ?? {}, makeCtx());
@@ -331,7 +372,7 @@ describe("agent config", () => {
 });
 
 describe("the transfer is terminal", () => {
-  test.each(registry)("%s refuses once the call is with a human", async (name, def) => {
+  test.each(gatedTools)("%s refuses once the call is with a human", async (name, def) => {
     const ctx = servingCtx();
     callFlow.send(ctx, { type: "TRANSFERRED" });
 

package/dist/templates/simple/agent.test.ts

@@ -2,17 +2,50 @@ import { toAgentConfig } from "@alexkroman1/aai/manifest";
 import { describe, expect, test } from "vitest";
 import agentDef from "./agent.ts";
 
+/**
+ * What a starter's spec may assert.
+ *
+ * Renaming the agent, giving it a voice, swapping a stage or switching the
+ * whole thing to speech-to-speech are the first edits this template invites —
+ * and `aai build` runs these tests before it bundles, so an assertion that
+ * pins the template's own identity turns the first customization into a build
+ * failure in a file the author never wrote. Every test here therefore asserts
+ * a property that survives those edits, on the RESOLVED config rather than on
+ * the def's empty fields.
+ */
 describe("simple template", () => {
   test("config passes manifest validation", () => {
     // Same conversion `aai build`/`aai deploy` run.
     expect(() => toAgentConfig(agentDef)).not.toThrow();
   });
 
-  test("exports an agent with a name and no explicit providers", () => {
-    // No provider fields declared: the default all-AssemblyAI pipeline is
-    // injected at parse time (see `defaultProviders`).
-    expect(agentDef.name).toBe("Simple Assistant");
-    expect(agentDef.stt).toBeUndefined();
-    expect(agentDef.s2s).toBeUndefined();
+  test("exports an agent the platform can name", () => {
+    // Not the literal: what has to hold is that there IS a name, and that the
+    // conversion carries it through — `AgentName` refuses a blank one, and the
+    // studio lists a deployed agent by exactly this string.
+    expect(agentDef.name).toBeTruthy();
+    expect(toAgentConfig(agentDef).name).toBe(agentDef.name);
+  });
+
+  test("every stage its mode needs is filled, declared or defaulted", () => {
+    // The template's point: with no provider fields declared, the default
+    // all-AssemblyAI cascaded pipeline is injected at parse time (see
+    // `defaultProviders`) — so an agent that declares nothing still runs.
+    // Asserted per MODE so it stays true after a swap: declare `stt`/`llm`/`tts`
+    // and the rest still default; declare `s2s` and there is no cascade to fill,
+    // which is the one thing that must never happen by fallthrough.
+    const config = toAgentConfig(agentDef);
+    if (config.mode === "s2s") {
+      expect(config.s2s?.kind).toBeTruthy();
+      expect(config.stt).toBeUndefined();
+      expect(config.tts).toBeUndefined();
+    } else if (config.mode === "text") {
+      expect(config.llm?.kind).toBeTruthy();
+    } else {
+      expect(config.mode).toBe("pipeline");
+      expect(config.stt?.kind).toBeTruthy();
+      expect(config.llm?.kind).toBeTruthy();
+      expect(config.tts?.kind).toBeTruthy();
+    }
   });
 });

package/dist/templates/spoken-summary/agent.eval.test.ts

@@ -48,6 +48,7 @@
 // exercised. `run.slept` below is the other half of that admission written as
 // an assertion. `aai-cli`'s `dev-workflow.scenario.test.ts` is the tier that
 // really suspends and resumes a run.
+import { stubGatewayRoute } from "@alexkroman1/aai/testing";
 import {
   installStubSpeech,
   installStubTranscribe,
@@ -115,23 +116,30 @@ function publish(bytes: Uint8Array, name: string, type: string) {
  *
  * ONE fake, because publishing a `stepFetch` REPLACES — a flow that transcribes
  * AND calls a model cannot install two, which is exactly what `otherwise` is
- * for. The transcription half is the SDK's own fake rather than this file's
- * hand-typed wire: it routes off the SDK's endpoint constants, so a case cannot
- * pass because the fake and the step agree on a typo.
+ * for. BOTH halves are the SDK's own fakes rather than this file's hand-typed
+ * wire, and it is the same argument twice: each routes off the SDK's own
+ * endpoint constant, so a case cannot pass because the fake and the step agree
+ * on a typo. The predicate here used to be `url.includes("llm-gateway")` — a
+ * HOST, which the default gateway happens to carry and a `gatewayUrl` pointed
+ * anywhere else does not, so the fake would have gone on answering the
+ * transcription 404 to a model call it no longer recognised.
+ *
+ * The reader also hands back DECODED calls, which is why the case below asks
+ * `model.calls[0].prompt` what the model was SHOWN: off a raw request body that
+ * is the whole serialized request, `model` and `reasoning_effort` included.
  */
 function scriptProvider(options: { text?: string; pendingPolls?: number } = {}) {
-  return installStubTranscribe({
+  const model = stubGatewayRoute(JSON.stringify(REPLY));
+  const provider = installStubTranscribe({
     text: options.text ?? TRANSCRIPT,
     durationSec: 42,
     // Passed straight through rather than conditionally spread: the option
     // already admits `undefined`, and `guard-invariants` rule 2 counts the
     // spread.
     pendingPolls: options.pendingPolls,
-    otherwise: (request) =>
-      request.url.includes("llm-gateway")
-        ? { body: { choices: [{ message: { content: JSON.stringify(REPLY) } }] } }
-        : undefined,
+    otherwise: (request) => model.route(request),
   });
+  return { provider, model };
 }
 
 describeWorkflowEval(
@@ -144,7 +152,7 @@ describeWorkflowEval(
       // returned bytes, or an id nothing wrote, or two ids because the synthesis
       // and the store became two steps.
       const uploads = publish(new Uint8Array(64), "standup.wav", "audio/wav");
-      const provider = scriptProvider();
+      const { provider } = scriptProvider();
       const speech = installStubSpeech({ pcmBytes: 96_000 });
 
       const run = await app.run(spokenSummary, { recording: UPLOAD_ID });
@@ -207,7 +215,7 @@ describeWorkflowEval(
       // central prompt decision regressing — synthesize the bullet list and you
       // get a voice reading "one. two. three." with no connective tissue.
       publish(new Uint8Array(64), "standup.wav", "audio/wav");
-      const provider = scriptProvider();
+      const { model } = scriptProvider();
       const speech = installStubSpeech();
 
       const run = await app.run(spokenSummary, { recording: UPLOAD_ID, voice: "michael" });
@@ -226,9 +234,10 @@ describeWorkflowEval(
 
       // And the model was ASKED for both, over the transcript it was given. A
       // prompt that stopped asking for a script is how the field goes missing.
-      const prompt = String(provider.calls.find((call) => call.leg === "other")?.body ?? "");
-      expect(prompt).toContain("READ ALOUD");
-      expect(prompt).toContain("The launch is on for Tuesday the fourth");
+      const asked = model.calls[0];
+      if (asked === undefined) expect.fail("the run must have asked the model for a summary");
+      expect(asked.prompt).toContain("READ ALOUD");
+      expect(asked.prompt).toContain("The launch is on for Tuesday the fourth");
     });
 
     test("a recording with no speech stops before the model and the voice", async ({ app }) => {
@@ -237,7 +246,7 @@ describeWorkflowEval(
       // the run would go on to summarize no words and store half a second of
       // audio — a green run with an empty product.
       const uploads = publish(new Uint8Array(64), "silence.wav", "audio/wav");
-      const provider = scriptProvider({ text: "   " });
+      const { model } = scriptProvider({ text: "   " });
       const speech = installStubSpeech();
 
       const run = await app.run(spokenSummary, { recording: UPLOAD_ID });
@@ -247,7 +256,7 @@ describeWorkflowEval(
       expect(run.output).toBeUndefined();
       // Nothing was summarized and nothing was spoken, which is the half that
       // makes this more than an error-message assertion.
-      expect(provider.calls.filter((call) => call.leg === "other")).toEqual([]);
+      expect(model.calls).toEqual([]);
       expect(speech.calls).toEqual([]);
       expect(uploads.writes).toEqual([]);
       expect(run.reported).not.toContain("Summarizing the transcript.");
@@ -262,7 +271,7 @@ describeWorkflowEval(
       // them. A loop that re-submitted, or one that spun with no wait, both
       // produce a correct transcript and a wrong bill.
       publish(new Uint8Array(64), "standup.wav", "audio/wav");
-      const provider = scriptProvider({ pendingPolls: 2 });
+      const { provider } = scriptProvider({ pendingPolls: 2 });
       installStubSpeech();
 
       const run = await app.run(spokenSummary, { recording: UPLOAD_ID });