@alexkroman1/aai-cli 13.0.0 → 13.2.0

package/dist/scaffold/package.json

@@ -14,9 +14,9 @@
     "publish:agent": "aai publish"
   },
   "dependencies": {
-    "@alexkroman1/aai": "^13.0.0",
-    "@alexkroman1/aai-runtime": "^13.0.0",
-    "@alexkroman1/aai-ui": "^13.0.0",
+    "@alexkroman1/aai": "^13.2.0",
+    "@alexkroman1/aai-runtime": "^13.2.0",
+    "@alexkroman1/aai-ui": "^13.2.0",
     "react": "^19.2.8",
     "react-dom": "^19.2.8",
     "tailwindcss": "^4.0.0",
@@ -24,7 +24,7 @@
     "zod": "^4.5.4"
   },
   "devDependencies": {
-    "@alexkroman1/aai-cli": "^13.0.0",
+    "@alexkroman1/aai-cli": "^13.2.0",
     "@tailwindcss/vite": "^4.3.3",
     "@types/node": "^26.4.1",
     "@types/react": "^19.2.18",

package/dist/templates/call-audit/agent.eval.test.ts

@@ -52,6 +52,7 @@
 // the tier that really resumes a run.
 import { spawnSync } from "node:child_process";
 import { encodeWav } from "@alexkroman1/aai/step";
+import { stubGatewayRoute } from "@alexkroman1/aai/testing";
 import { installStubTranscribe, installStubUploads } from "@alexkroman1/aai/testing/vitest";
 import { describeWorkflowEval } from "@alexkroman1/aai-runtime/eval/vitest";
 import { describe, expect, test } from "vitest";
@@ -211,19 +212,21 @@ function publish(bytes: Uint8Array, name: string, type: string) {
  * Answer the sync transcription endpoint and the model in memory.
  *
  * ONE fake, because publishing a `stepFetch` REPLACES — a flow that transcribes
- * AND calls a model cannot install two, which is what `otherwise` is for. The
- * transcription half is the SDK's own fake rather than this file's hand-typed
- * wire: it routes off the SDK's endpoint constants, so a case cannot pass
- * because the fake and the step agree on a typo.
+ * AND calls a model cannot install two, which is what `otherwise` is for. BOTH
+ * halves are the SDK's own fakes rather than this file's hand-typed wire, and
+ * for the same reason: each routes off the SDK's own endpoint constant, so a
+ * case cannot pass because the fake and the step agree on a typo. The gateway
+ * envelope is the half where that matters most — it is a WIRE shape, so getting
+ * a field wrong does not fail: `stepGenerate` reads no content, reports an empty
+ * completion, and the case blames the desk.
+ *
+ * `route` answers `undefined` for anything that is not a completion request, so
+ * it drops straight into `otherwise` and the transcription legs still reach the
+ * fake below it.
  */
 function scriptProvider(text: readonly string[]) {
-  return installStubTranscribe({
-    text,
-    otherwise: (request) =>
-      request.url.includes("/chat/completions")
-        ? { body: { choices: [{ message: { content: JSON.stringify(REPLY) } }] } }
-        : undefined,
-  });
+  const model = stubGatewayRoute(JSON.stringify(REPLY));
+  return installStubTranscribe({ text, otherwise: (request) => model.route(request) });
 }
 
 describeWorkflowEvalWithFfmpeg(

package/dist/templates/call-audit/agent.test.ts

@@ -928,9 +928,21 @@ describe("the body's step policy", () => {
  *
  * The block above drives the body through `createWorkflowCtx` with
  * `runSteps: false` and a journaled result per step, which is what makes it
- * affordable: `ingestRecording` runs ffmpeg, and this repo's test environment
- * has none — the `ingestRecording` and `narrate` specs above assert the
- * `FatalError` that produces, unconditionally.
+ * affordable: `ingestRecording` shells out to the ffmpeg toolchain, which this
+ * tier cannot usefully feed — the `ingestRecording` and `narrate` specs above
+ * assert the `FatalError` that produces, unconditionally.
+ *
+ * **The run fails here whether or not ffmpeg is installed, and the two failures
+ * do not say the same thing** — which is what the assertion below has to be
+ * written against. With no binary on `PATH` the step reports the missing
+ * toolchain as an instruction (the template's own doc promises that rather than
+ * `spawn ffmpeg ENOENT`); WITH one, the step gets a step further and `ffprobe`
+ * rejects the 2 KB stub above with `ffprobe exited with code 1`. This docblock
+ * used to state "this repo's test environment has none" as a premise, and the
+ * spec below matched `/ffmpeg/i` on the strength of it — so the test passed only
+ * on a machine where ffmpeg was ABSENT, and a developer with Homebrew's ffmpeg
+ * saw a red suite on a clean checkout of `main`. CI has no ffmpeg, so nothing
+ * caught it.
  *
  * So a whole run of this desk is not reachable here and this file does not
  * pretend otherwise; `aai-cli`'s `dev-workflow.scenario.test.ts` is the tier
@@ -953,7 +965,11 @@ describe("the run is DURABLE, as far as ffmpeg allows", () => {
     const run = await runWorkflow(audit, { recording: UPLOAD_ID }, { name: "audit" });
 
     expect(run.status).toBe("failed");
-    expect(run.error).toMatch(/ffmpeg/i);
+    // Either member of the toolchain, for the reason this block's doc gives: the
+    // step names `ffmpeg` when the binary is missing and `ffprobe` when it is
+    // present, and this spec is about neither. What it is about is the two
+    // assertions below — the ENGINE honouring `FatalError`.
+    expect(run.error).toMatch(/ff(mpeg|probe)/i);
     const ingest = run.steps.find((step) => step.name === "ingestRecording");
     // The whole point of `FatalError`: `maxAttempts: 6` is the budget this call
     // site asks for, and a failure that cannot change must not spend it. A step

package/dist/templates/call-audit/workflows/ingest.ts

@@ -143,7 +143,16 @@ export async function ingestRecording(uploadId: string): Promise<Ingested> {
       const normalized = join(dir, "audio.pcm");
       const silenceLog = join(dir, "silence.txt");
 
-      await readUploadToFile(uploadId, source, { size: stored.size });
+      // NO `size`, though `stored.size` is right there — and that is the whole
+      // difference between this copy being one window at a time and being
+      // `STEP_FILE_READ_CONCURRENCY` of them. Passing `size` means "I am judging
+      // completeness myself", which is what a body polling a still-arriving
+      // upload needs and is the opposite of what happened above: this step has
+      // already called `requireCompleteUpload`, so the file IS whole and the
+      // windows may land in any order. Omitting it lets `readUploadToFile`
+      // establish that for itself and fan out. The cost is one metadata round
+      // trip, against the dozens of window reads it overlaps.
+      await readUploadToFile(uploadId, source);
 
       // What it WAS, for the progress log and the page. Worth one ffprobe: "41
       // minutes of aac" explains the shape of the run, where "the recording" leaves

package/dist/templates/code-interpreter/agent.eval.test.ts

@@ -24,27 +24,37 @@
 // Coda reached for code, and what the code came back with.
 
 import agentDef from "virtual:aai/agent";
-import { createVmRunCode, toolResultsIn } from "@alexkroman1/aai-runtime/eval";
+import {
+  createVmRunCode,
+  type EvalTurn,
+  toolArgsIn,
+  toolNames,
+  toolResultsIn,
+} from "@alexkroman1/aai-runtime/eval";
 import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
 import { expect } from "vitest";
+import { z } from "zod";
 
-/** The code every `run_code` call in this turn carried, joined. */
-const codeIn = (turn: { toolCalls: readonly { name: string; args: Record<string, unknown> }[] }) =>
-  turn.toolCalls
-    .filter((c) => c.name === "run_code")
-    .map((c) => String(c.args.code ?? ""))
+/**
+ * The code every `run_code` call in this turn carried, joined.
+ *
+ * Read through `toolArgsIn` WITH a schema, which is what that reader takes one
+ * for: `args` is `Record<string, unknown>` on the wire — the model wrote it and
+ * nothing validated it — so the `String(c.args.code ?? "")` this replaced turned
+ * an argument Coda renamed, or never sent, into `""`, and every claim below about
+ * the code she wrote would have been a claim about an empty string. A `code`
+ * that stops arriving FAILS here, naming the field.
+ */
+const RunCodeArgs = z.object({ code: z.string() });
+const codeIn = (turn: EvalTurn) =>
+  toolArgsIn(turn.toolCalls, "run_code", RunCodeArgs)
+    .map((args) => args.code)
     .join("\n");
 
 /**
- * A `run_code` executor, so these cases can assert the ANSWER.
- *
- * The builtin refuses without one — the Modal container is the security
- * boundary, and off-platform there is none — so a case could assert the CALL and
- * the code it carried, and never what the code came back with.
- * `createVmRunCode()` is a `node:vm` context with a capturing `console.log`,
- * which is enough here: what runs is arithmetic, not a program. It is NOT a
- * sandbox and does not pretend to be one; a deployed agent still gets the
- * refusal.
+ * A `run_code` executor, so these cases can assert the ANSWER and not merely the
+ * call — `createVmRunCode`'s own doc carries why the builtin refuses without one
+ * and why a `node:vm` context is the right thing to hand it here.
  */
 const runCode = createVmRunCode();
 
@@ -59,7 +69,7 @@ describeEval(
         // The template's CRITICAL RULE, and the whole reason it declares
         // run_code: a model that answers this one directly has regressed, and it
         // is the easiest question in the file to answer wrongly with confidence.
-        expect(turn.toolCalls.map((c) => c.name)).toContain("run_code");
+        expect(toolNames(turn.toolCalls)).toContain("run_code");
         const code = codeIn(turn);
         expect(code).toContain("127");
         expect(code).toContain("849");
@@ -90,7 +100,7 @@ describeEval(
         // The prompt lists this exact question under "you MUST use code for".
         // It is the case a narrower reading of the rule ("code is for maths")
         // silently drops.
-        expect(turn.toolCalls.map((c) => c.name)).toContain("run_code");
+        expect(toolNames(turn.toolCalls)).toContain("run_code");
         expect(codeIn(turn)).toMatch(/Date|2000/);
         // The code RAN rather than being refused — but the ANSWER is
         // deliberately not asserted here, and the reason is worth knowing before

package/dist/templates/dispatch-center/agent.eval.test.ts

@@ -34,7 +34,12 @@ import type { SessionEvent } from "@alexkroman1/aai/protocol";
 // What no eval here can see: anything below the audio boundary. Whether a
 // dispatcher reading a callsign in bursts lands as one turn is a property of
 // endpointing, and these fake speech stages remove it.
-import { type EvalSession, type EvalTurn, lastStateIn } from "@alexkroman1/aai-runtime/eval";
+import {
+  type EvalSession,
+  lastStateIn,
+  toolNames,
+  turnCalling,
+} from "@alexkroman1/aai-runtime/eval";
 import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
 import { expect } from "vitest";
 import { z } from "zod";
@@ -100,20 +105,6 @@ const boardEntry = (events: readonly SessionEvent[], id: string) =>
 const refusalAt = (state: string) =>
   new RegExp(`Not available yet: this conversation is at [\\\\"]*${state}`);
 
-/**
- * Drive a whole shift, one dispatcher line at a time, and hand back every turn.
- *
- * The cases below assert about the turn a MECHANISM fired in rather than about
- * turn number two: how many turns a desk takes to get there is the model's
- * business and it varies, and a case pinned to a turn index is a flake with a
- * misleading name.
- */
-async function sayAll(session: EvalSession, lines: readonly string[]): Promise<EvalTurn[]> {
-  const turns: EvalTurn[] = [];
-  for (const line of lines) turns.push(await session.say(line));
-  return turns;
-}
-
 /** Every call to `tool` across the whole shift. */
 const callsTo = (session: EvalSession, tool: string) =>
   session.toolCalls().filter((c) => c.name === tool);
@@ -200,12 +191,15 @@ describeEval(dispatchAgent, (test) => {
   test(
     "rolls units on a logged incident and follows them to monitoring",
     async ({ session }) => {
-      const turns = await sayAll(session, [THE_CALL, "Dispatch the recommended units now."]);
+      const turns = await session.sayAll([THE_CALL, "Dispatch the recommended units now."]);
 
-      const dispatching = turns.find((t) =>
-        t.toolCalls.some((c) => c.name === "resources_dispatch"),
-      );
-      const rolled = dispatching?.toolCalls.find((c) => c.name === "resources_dispatch");
+      // The turn the dispatch fired in, whichever one that turned out to be:
+      // how many turns a desk spends getting there is the model's business, so
+      // a case pinned to turn two is a flake with a misleading name. A shift
+      // that never dispatched at all fails HERE, with every turn's tool list in
+      // the message, rather than as an `undefined` three assertions later.
+      const dispatching = turnCalling(turns, "resources_dispatch");
+      const rolled = dispatching.toolCalls.find((c) => c.name === "resources_dispatch");
       // Units really assigned — `dispatched` is empty when every requested
       // callsign was busy, which is the case the fourth test owns.
       expect(rolled?.result).toMatch(/"dispatched":\[\{/);
@@ -215,7 +209,7 @@ describeEval(dispatchAgent, (test) => {
       // The board agrees: the incident is dispatched, not merely triaged.
       expect(boardEntry(session.events(), FIRST_INCIDENT)?.status).toBe("dispatched");
       // And the order is the one the desk's flow requires: log, then dispatch.
-      const names = session.toolCalls().map((c) => c.name);
+      const names = toolNames(session.toolCalls());
       expect(names.indexOf("resources_dispatch")).toBeGreaterThan(names.indexOf("incident_create"));
     },
     {
@@ -230,7 +224,7 @@ describeEval(dispatchAgent, (test) => {
   test(
     "closing an incident releases the units that were on it",
     async ({ session }) => {
-      await sayAll(session, [
+      await session.sayAll([
         THE_CALL,
         "Dispatch the recommended units, emergency priority.",
         "Units report the patient is transported and they're clear. Close it out.",
@@ -252,7 +246,7 @@ describeEval(dispatchAgent, (test) => {
       // The board agrees, which is the half a browser would show.
       expect(boardEntry(session.events(), FIRST_INCIDENT)?.status).toBe("resolved");
       // And the shift ran in the order the flow requires.
-      const names = session.toolCalls().map((c) => c.name);
+      const names = toolNames(session.toolCalls());
       expect(names.indexOf("incident_create")).toBeGreaterThanOrEqual(0);
       expect(names.indexOf("resources_dispatch")).toBeGreaterThan(names.indexOf("incident_create"));
       expect(names.lastIndexOf("incident_update_status")).toBeGreaterThan(
@@ -279,7 +273,7 @@ describeEval(dispatchAgent, (test) => {
   test(
     "a unit already rolling is not sent to a second call",
     async ({ session }) => {
-      await sayAll(session, [
+      await session.sayAll([
         THE_CALL,
         "Send Medic-1 to Oak Street.",
         "New call: 12 Pine Lane, chest pains. Log it.",

package/dist/templates/embedded-assets/agent.eval.test.ts

@@ -25,7 +25,7 @@
  * tool-choice claim below then passes or fails for the wrong reason.
  */
 import agentDef from "virtual:aai/agent";
-import { toolResultIn } from "@alexkroman1/aai-runtime/eval";
+import { toolNames, toolResultIn } from "@alexkroman1/aai-runtime/eval";
 import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
 import { expect } from "vitest";
 import { z } from "zod";
@@ -57,7 +57,7 @@ describeEval(agentDef, (test) => {
       // about voice frameworks in general.
       const turn = await session.say("Can your agents make HTTP requests?");
 
-      expect(turn.toolCalls.map((c) => c.name)).toEqual(["search_knowledge"]);
+      expect(toolNames(turn.toolCalls)).toEqual(["search_knowledge"]);
       const call = turn.toolCalls[0]!;
       expect(typeof (call.args as { query?: unknown }).query).toBe("string");
       // The right entry, out of four: the one this question is really about.
@@ -81,7 +81,7 @@ describeEval(agentDef, (test) => {
     async ({ session }) => {
       const turn = await session.say("What topics can you help me with?");
 
-      expect(turn.toolCalls.map((c) => c.name)).toEqual(["list_topics"]);
+      expect(toolNames(turn.toolCalls)).toEqual(["list_topics"]);
       // Every question in `knowledge.json` and nothing else — the check that
       // catches an index built from a stale copy of the asset.
       expect(toolResultIn(turn.toolCalls, "list_topics", z.array(z.string()))).toEqual(

package/dist/templates/health-assistant/agent.eval.test.ts

@@ -34,14 +34,36 @@
  * tool-choice claim below then passes or fails for the wrong reason.
  */
 import agentDef from "virtual:aai/agent";
-import { toolResultIn } from "@alexkroman1/aai-runtime/eval";
+import {
+  describeTurn,
+  type EvalToolCall,
+  toolArgsIn,
+  toolResultIn,
+} from "@alexkroman1/aai-runtime/eval";
 import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
 import { expect } from "vitest";
 import { z } from "zod";
 
-/** The drugs an interaction check was actually asked about, lowercased. */
-const drugsIn = (args: Record<string, unknown>): string[] =>
-  ((args.drugs as string[] | undefined) ?? []).map((d) => d.trim().toLowerCase());
+/**
+ * Every drug this scope's interaction checks were actually asked about,
+ * lowercased.
+ *
+ * Read through `toolArgsIn` WITH a schema, which is what that reader takes one
+ * for: `args` is `Record<string, unknown>` on the wire — the model wrote it and
+ * nothing validated it — so the `args.drugs as string[] ?? []` this replaced
+ * turned a `drugs` the desk renamed, or never sent, into an empty list, and the
+ * two claims below would have been claims about nothing. A `drugs` that stops
+ * arriving FAILS here, naming the field.
+ *
+ * ZERO checks answers `[]` rather than throwing, which is what keeps the
+ * dangerous case assertable: a desk that answered an interaction question
+ * without asking a single label reaches the assertions with nothing in hand.
+ */
+const InteractionArgs = z.object({ drugs: z.array(z.string()) });
+const drugsAsked = (calls: readonly EvalToolCall[]): string[] =>
+  toolArgsIn(calls, "check_drug_interaction", InteractionArgs).flatMap((args) =>
+    args.drugs.map((drug) => drug.trim().toLowerCase()),
+  );
 
 /**
  * A refusal from `check_drug_interaction`, as the model saw it.
@@ -60,12 +82,15 @@ describeEval(agentDef, (test) => {
       const turn = await session.say("Can I take ibuprofen and warfarin together?");
 
       // It may look each drug up as well — that is fine and often useful. What
-      // it may not do is answer this question without asking the labels.
-      const checks = turn.toolCalls.filter((c) => c.name === "check_drug_interaction");
-      expect(checks).not.toEqual([]);
-      const asked = checks.flatMap((c) => drugsIn(c.args));
-      expect(asked).toContain("ibuprofen");
-      expect(asked).toContain("warfarin");
+      // it may not do is answer this question without asking the labels — and a
+      // desk that asked nothing arrives here with an empty list, so these two
+      // lines carry the never-checked finding as well as the wrong-drugs one.
+      // `describeTurn` is what tells them apart in the failure: "expected [] to
+      // contain 'ibuprofen'" does not say whether the desk called something
+      // else, called nothing, or was cut off mid-reply.
+      const asked = drugsAsked(turn.toolCalls);
+      expect(asked, describeTurn(turn)).toContain("ibuprofen");
+      expect(asked, describeTurn(turn)).toContain("warfarin");
 
       // The rule at the top of its prompt: it is not a doctor. An interaction
       // answer that does not end at a professional is the failure that makes
@@ -90,11 +115,9 @@ describeEval(agentDef, (test) => {
       // drug alone and report nothing.
       const turn = await session.say("Is it okay if I add ibuprofen for a headache?");
 
-      const checks = turn.toolCalls.filter((c) => c.name === "check_drug_interaction");
-      expect(checks).not.toEqual([]);
-      const asked = checks.flatMap((c) => drugsIn(c.args));
-      expect(asked).toContain("warfarin");
-      expect(asked).toContain("ibuprofen");
+      const asked = drugsAsked(turn.toolCalls);
+      expect(asked, describeTurn(turn)).toContain("warfarin");
+      expect(asked, describeTurn(turn)).toContain("ibuprofen");
     },
     {
       stubReply: [

package/dist/templates/link-digest/agent.eval.test.ts

@@ -26,15 +26,13 @@
 
 import { createServer, type Server } from "node:http";
 import type { AddressInfo } from "node:net";
+import { stubGatewayRoute } from "@alexkroman1/aai/testing";
 import { installStubStepFetch } from "@alexkroman1/aai/testing/vitest";
 import { describeWorkflowEval } from "@alexkroman1/aai-runtime/eval/vitest";
 import { expect, onTestFinished } from "vitest";
 import agentDef, { digest } from "./agent.ts";
 import { SETTLE_MS } from "./workflows/digest.ts";
 
-/** The gateway leg, so one handler can route the page and the model apart. */
-const isModelCall = (url: string): boolean => url.includes("/chat/completions");
-
 /**
  * A page with an ANSWER in it, so "did it summarize what it fetched" is a
  * question with a right answer rather than a vibe.
@@ -107,15 +105,25 @@ async function servePage(html: string): Promise<string> {
  *
  * ONE handler, because publishing a `stepFetch` REPLACES — a flow that fetches a
  * page and calls a model cannot install two fakes, so it routes by URL. The
- * recorded calls are what makes the prompt assertable, which is the only way to
- * check what the model was SHOWN rather than what it said.
+ * routing is `stubGatewayRoute`'s rather than this file's: it answers the
+ * completion request and `undefined` for everything else, which is what makes
+ * the page the `??` arm — and it routes off the SDK's own completions PATH, so a
+ * case cannot pass because the fake and the step agree on a typo. That matters
+ * more here than it looks: the envelope is a WIRE shape, so a field typed one
+ * off does not fail — `stepGenerate` reads no content and reports an empty
+ * completion, and the case blames the digest.
+ *
+ * The recorded calls are what makes the prompt assertable, which is the only way
+ * to check what the model was SHOWN rather than what it said — and they come
+ * back DECODED, so the last case reads `prompt` rather than the raw request
+ * body, which is the whole serialized request.
  */
 function scriptBothLegs(html: string, reply = SCRIPTED_DIGEST) {
-  return installStubStepFetch((request) =>
-    isModelCall(request.url)
-      ? { body: { choices: [{ message: { content: reply } }] } }
-      : { body: html, headers: { "Content-Type": "text/html" } },
+  const model = stubGatewayRoute(reply);
+  installStubStepFetch(
+    (request) => model.route(request) ?? { body: html, headers: { "Content-Type": "text/html" } },
   );
+  return model;
 }
 
 describeWorkflowEval(agentDef, (test) => {
@@ -180,18 +188,19 @@ describeWorkflowEval(agentDef, (test) => {
       "</body>",
       `<p>${"padding sentence about otters. ".repeat(2000)}</p></body>`,
     );
-    const legs = scriptBothLegs(oversized);
+    const model = scriptBothLegs(oversized);
 
     const run = await app.run(digest, { url: "https://example.test/otters" });
     expect(run.status).toBe("completed");
 
-    const prompt = String(legs.calls.find((call) => isModelCall(call.url))?.body ?? "");
-    expect(prompt).toContain("Sea otters are one of the few mammals that use tools");
+    const asked = model.calls[0];
+    if (asked === undefined) expect.fail("the run must have shown the article to the model");
+    expect(asked.prompt).toContain("Sea otters are one of the few mammals that use tools");
     // Neither the script's instruction nor the stylesheet reached the prompt.
-    expect(prompt).not.toContain(SMUGGLED);
-    expect(prompt).not.toContain("rebeccapurple");
+    expect(asked.prompt).not.toContain(SMUGGLED);
+    expect(asked.prompt).not.toContain("rebeccapurple");
     // And the text was CAPPED on the way across the queue. 24k characters plus
     // the prompt's own framing, well under the ~60k this page would otherwise be.
-    expect(prompt.length).toBeLessThan(30_000);
+    expect(asked.prompt.length).toBeLessThan(30_000);
   });
 });

package/dist/templates/math-buddy/agent.eval.test.ts

@@ -14,27 +14,38 @@
 // back with as well as the code the tutor wrote.
 
 import agentDef from "virtual:aai/agent";
-import { createVmRunCode, toolResultsIn } from "@alexkroman1/aai-runtime/eval";
+import {
+  createVmRunCode,
+  type EvalTurn,
+  toolArgsIn,
+  toolNames,
+  toolResultsIn,
+} from "@alexkroman1/aai-runtime/eval";
 import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
 import { expect } from "vitest";
+import { z } from "zod";
 
-/** The code every `run_code` call in this turn carried, joined. */
-const codeIn = (turn: { toolCalls: readonly { name: string; args: Record<string, unknown> }[] }) =>
-  turn.toolCalls
-    .filter((c) => c.name === "run_code")
-    .map((c) => String(c.args.code ?? ""))
+/**
+ * The code every `run_code` call in this turn carried, joined.
+ *
+ * Read through `toolArgsIn` WITH a schema, which is what that reader takes one
+ * for: `args` is `Record<string, unknown>` on the wire — the tutor wrote it and
+ * nothing validated it — so the `String(c.args.code ?? "")` this replaced turned
+ * an argument the model renamed, or never sent, into `""`. The claims below are
+ * about the recipe the tutor wrote, and against `""` every one of them would
+ * have been a claim about nothing. A `code` that stops arriving FAILS here,
+ * naming the field.
+ */
+const RunCodeArgs = z.object({ code: z.string() });
+const codeIn = (turn: EvalTurn) =>
+  toolArgsIn(turn.toolCalls, "run_code", RunCodeArgs)
+    .map((args) => args.code)
     .join("\n");
 
 /**
- * A `run_code` executor, so these cases can assert the ANSWER.
- *
- * The builtin refuses without one — the Modal container is the security
- * boundary, and off-platform there is none — so a case could assert the CALL and
- * the code it carried, and never what the code came back with.
- * `createVmRunCode()` is a `node:vm` context with a capturing `console.log`,
- * which is enough here: what runs is arithmetic, not a program. It is NOT a
- * sandbox and does not pretend to be one; a deployed agent still gets the
- * refusal.
+ * A `run_code` executor, so these cases can assert the answer the tutor's code
+ * came back with and not merely the code — `createVmRunCode`'s own doc carries
+ * why the builtin refuses without one.
  */
 const runCode = createVmRunCode();
 
@@ -49,7 +60,7 @@ describeEval(
         // The prompt hands the tutor the factors; the finding it guards against
         // is a tutor that recites a remembered figure instead. A factor in the
         // code is the evidence that the conversion was computed, not recalled.
-        expect(turn.toolCalls.map((c) => c.name)).toContain("run_code");
+        expect(toolNames(turn.toolCalls)).toContain("run_code");
         const code = codeIn(turn);
         expect(code).toContain("5");
         expect(code).toMatch(/1\.60|1\.61|0\.621|8\.04/);
@@ -69,7 +80,7 @@ describeEval(
         // A model asked for dice will happily make three numbers up, and the
         // reply is indistinguishable from a real roll. `Math.random` in the code
         // is the only thing that tells them apart.
-        expect(turn.toolCalls.map((c) => c.name)).toContain("run_code");
+        expect(toolNames(turn.toolCalls)).toContain("run_code");
         const code = codeIn(turn);
         expect(code).toMatch(/Math\.random/);
         expect(code).toContain("20");

package/dist/templates/night-owl/agent.eval.test.ts

@@ -29,7 +29,10 @@ import type { SessionEvent } from "@alexkroman1/aai/protocol";
 import {
   createVmRunCode,
   customEventsIn,
+  describeTurn,
   lastStateIn,
+  toolArgsIn,
+  toolNames,
   toolResultIn,
 } from "@alexkroman1/aai-runtime/eval";
 import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
@@ -66,17 +69,23 @@ const pushedRecs = (events: readonly SessionEvent[]) =>
 const nudges = (events: readonly SessionEvent[]) => customEventsIn(events, "wind_down");
 
 /**
- * A `run_code` executor, so the sleep-cycle case can assert the ANSWER.
- *
- * The builtin refuses without one — the Modal container is the security
- * boundary, and off-platform there is none — which left this template's
- * headline feature assertable as a CALL and never as a number. A `node:vm`
- * context with a capturing `console.log` is what a developer would reach for on
- * their own machine, and `createVmRunCode()` is exactly that: the code under
- * test is arithmetic the model wrote, not a program.
+ * A `run_code` executor, so the sleep-cycle case can assert the bedtime NUMBER
+ * and not merely the call — this template's headline feature is the arithmetic.
+ * `createVmRunCode`'s own doc carries why the builtin refuses without one.
  */
 const runCode = createVmRunCode();
 
+/**
+ * The `code` argument a `run_code` call carries.
+ *
+ * The schema is what `toolArgsIn` takes one for: `args` is
+ * `Record<string, unknown>` on the wire — the model wrote it and nothing
+ * validated it — so the `String(c.args.code ?? "")` this replaced turned an
+ * argument the companion renamed, or never sent, into `""`, and the two
+ * constants asserted below would have been looked for in nothing at all.
+ */
+const RunCodeArgs = z.object({ code: z.string() });
+
 /** Two-digit, for the clock arithmetic below. */
 const pad = (n: number): string => String(n).padStart(2, "0");
 
@@ -113,7 +122,7 @@ describeEval(
 
         // "to watch" is the category and "cozy" is the mood; the shelf is the
         // tool's, so answering from the model's own taste is the regression.
-        expect(turn.toolCalls.map((c) => c.name)).toEqual(["recommend"]);
+        expect(toolNames(turn.toolCalls)).toEqual(["recommend"]);
         const call = turn.toolCalls[0]!;
         expect(call.args).toEqual({ category: "movie", mood: "cozy" });
 
@@ -139,7 +148,7 @@ describeEval(
         await session.say("I want something cozy to watch tonight.");
         const turn = await session.say("Now give me something spooky to read.");
 
-        expect(turn.toolCalls.map((c) => c.name)).toEqual(["recommend"]);
+        expect(toolNames(turn.toolCalls)).toEqual(["recommend"]);
         expect(turn.toolCalls[0]!.args).toEqual({ category: "book", mood: "spooky" });
 
         // The slot survived the turn boundary: the frame the page renders after
@@ -197,15 +206,21 @@ describeEval(
           "I need to be up at 7 in the morning. When should I fall asleep?",
         );
 
+        // The CALLS, not their arguments: what this asserts is that the companion
+        // reached for code at all, and the results are read off the same list
+        // below. `toolArgsIn` answers the other half, the code it submitted.
         const ran = turn.toolCalls.filter((c) => c.name === "run_code");
-        expect(
-          ran,
-          `tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
-        ).not.toEqual([]);
+        // `describeTurn` is the message: "expected [] not to equal []" says
+        // nothing about a companion that talked its way through the sum
+        // instead, and it names a cancelled reply, which is the usual reason a
+        // turn reached for nothing at all.
+        expect(ran, describeTurn(turn)).not.toEqual([]);
         // The recipe is the prompt's, and it is two constants: a 90-minute cycle
         // plus the 15 minutes it takes to fall asleep. Arithmetic done in the
         // model's head has neither of them anywhere in the code.
-        const code = ran.map((c) => String(c.args.code ?? "")).join("\n");
+        const code = toolArgsIn(turn.toolCalls, "run_code", RunCodeArgs)
+          .map((args) => args.code)
+          .join("\n");
         expect(code).toContain("90");
         expect(code).toContain("15");