@alexkroman1/aai-cli 7.0.0 → 8.1.0

package/dist/templates/math-buddy/agent.eval.test.ts

@@ -0,0 +1,146 @@
+// An EVAL: does Math Buddy delegate every calculation to code? Run it with
+// `aai eval`.
+//
+// This tutor's whole design is "the model does the talking, run_code does the
+// arithmetic" — which is also why it runs on Flash-Lite. So the claim worth
+// pinning is not that the answer is right, it is that the answer came from
+// CODE, and that the code is the recipe the prompt gave.
+//
+// Two things this file has to work around, both explained at length in
+// `../code-interpreter/agent.eval.test.ts`: `system-prompt.md` is discovered by
+// the build rather than imported, so an eval has to apply it or it measures a
+// different agent; and `run_code` refuses unless the EVAL supplies an executor,
+// which this suite does — so the cases below assert the answer the code came
+// back with as well as the code the tutor wrote.
+import { runInNewContext } from "node:vm";
+import { withSystemPrompt } from "@alexkroman1/aai/manifest";
+import { errorMessage } from "@alexkroman1/aai/utils";
+import type { RunCodeExecutor } from "@alexkroman1/aai-runtime/eval";
+import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import authored from "./agent.ts";
+import systemPrompt from "./system-prompt.md?raw";
+
+const agentDef = withSystemPrompt(authored, systemPrompt);
+
+/** The code every `run_code` call in this turn carried, joined. */
+const codeIn = (turn: { toolCalls: readonly { name: string; args: Record<string, unknown> }[] }) =>
+  turn.toolCalls
+    .filter((c) => c.name === "run_code")
+    .map((c) => String(c.args.code ?? ""))
+    .join("\n");
+
+/**
+ * A `run_code` executor, so these cases can assert the ANSWER.
+ *
+ * The builtin refuses without one — the Modal container is the security
+ * boundary, and off-platform there is none — so a case could assert the CALL and
+ * the code it carried, and never what the code came back with. A `node:vm`
+ * context with a capturing `console.log` is what a developer would reach for on
+ * their own machine, and it is enough here: what runs is arithmetic, not a
+ * program. It is NOT a sandbox and does not pretend to be one; a deployed agent
+ * still gets the refusal.
+ *
+ * A template eval imports from `@alexkroman1/aai-runtime/eval` and
+ * `/eval/vitest` and nowhere else — the ROOT barrel drags the host runtime's
+ * node-reaching module graph into this project's TypeScript program, which is
+ * three errors in runtime files no eval ever calls. `RunCodeExecutor` is
+ * re-exported from `/eval` for exactly that reason.
+ */
+const runCode: RunCodeExecutor = async (code) => {
+  const lines: string[] = [];
+  const log = (...args: unknown[]): void => {
+    lines.push(args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" "));
+  };
+  try {
+    runInNewContext(code, { console: { log } }, { timeout: 1000 });
+  } catch (err) {
+    return { error: errorMessage(err) };
+  }
+  return lines.join("\n");
+};
+
+/** What every `run_code` call in this turn PRINTED, joined. */
+const outputIn = (turn: { toolCalls: readonly { name: string; result?: string }[] }) =>
+  turn.toolCalls
+    .filter((c) => c.name === "run_code")
+    .map((c) => c.result ?? "")
+    .join("\n");
+
+describeEval(
+  agentDef,
+  (test) => {
+    test(
+      "converts units in code, with a real conversion factor",
+      async ({ session }) => {
+        const turn = await session.say("Convert 5 miles to kilometres.");
+
+        // The prompt hands the tutor the factors; the finding it guards against
+        // is a tutor that recites a remembered figure instead. A factor in the
+        // code is the evidence that the conversion was computed, not recalled.
+        expect(turn.toolCalls.map((c) => c.name)).toContain("run_code");
+        const code = codeIn(turn);
+        expect(code).toContain("5");
+        expect(code).toMatch(/1\.60|1\.61|0\.621|8\.04/);
+        // And the factor was applied rather than merely mentioned: five miles is
+        // 8.0467 km, so whatever rounding the tutor chose the answer starts 8.0.
+        const output = outputIn(turn);
+        expect(output, `run_code printed: ${output}`).toMatch(/8\.0/);
+      },
+      { live: true },
+    );
+
+    test(
+      "rolls dice with a random draw rather than inventing numbers",
+      async ({ session }) => {
+        const turn = await session.say("Roll 3 twenty-sided dice for me.");
+
+        // A model asked for dice will happily make three numbers up, and the
+        // reply is indistinguishable from a real roll. `Math.random` in the code
+        // is the only thing that tells them apart.
+        expect(turn.toolCalls.map((c) => c.name)).toContain("run_code");
+        const code = codeIn(turn);
+        expect(code).toMatch(/Math\.random/);
+        expect(code).toContain("20");
+        expect(code).toContain("3");
+
+        // And the draw really happened: three integers, every one of them a legal
+        // face of a twenty-sided die. `Math.random` in the code says the tutor
+        // asked for a roll; this says it GOT one — a `run_code` that refused
+        // prints a sentence with no dice in it at all.
+        const output = outputIn(turn);
+        const rolled = [...output.matchAll(/\d+/g)].map((m) => Number(m[0]));
+        expect(rolled.length, `run_code printed: ${output}`).toBeGreaterThanOrEqual(3);
+        for (const face of rolled) {
+          expect(face, `run_code printed: ${output}`).toBeGreaterThanOrEqual(1);
+          expect(face, `run_code printed: ${output}`).toBeLessThanOrEqual(20);
+        }
+      },
+      { live: true },
+    );
+
+    test(
+      "the run_code builtin is wired to the agent's tool executor",
+      async ({ session }) => {
+        const turn = await session.say("What is 127 times 849?");
+
+        // A tool the agent does not declare produces a `tool.called` with no
+        // result, so the paired result is what says `builtinTools: ["run_code"]`
+        // still resolves to something executable. The ANSWER rather than
+        // `toBeDefined()`, which the refusal string satisfied too.
+        const [call] = turn.toolCalls;
+        expect(call?.name).toBe("run_code");
+        expect(call?.result).toBe("107823");
+        expect(turn.completed).toBe(true);
+      },
+      {
+        stubReply: [
+          { tool: "run_code", args: { code: "console.log(127 * 849)" } },
+          "That's 107,823.",
+        ],
+      },
+    );
+  },
+  // `runCode` is what makes these cases about the ANSWER and not just the call.
+  { runCode },
+);

package/dist/templates/night-owl/agent.eval.test.ts

@@ -0,0 +1,272 @@
+/// <reference types="vite/client" />
+
+// An EVAL: does the companion actually pick from its own shelf?
+//
+// `agent.test.ts` calls `recommend` directly, which settles what the tool does
+// with a category and a mood it is handed. What it cannot settle is whether the
+// MODEL turns "something cozy to watch" into `{ category: "movie", mood:
+// "cozy" }` rather than reciting three films it likes — and whether the night's
+// log, which lives in a `sessionSlot`, is still there two turns later.
+//
+// Run it with `aai eval`. Without a provider key every case runs against a
+// SCRIPTED model (its `stubReply`): the real session, the real slot, the real
+// tool, a fake reply. That proves the wiring and nothing about the choice.
+
+import { runInNewContext } from "node:vm";
+import { withSystemPrompt } from "@alexkroman1/aai/manifest";
+import type { SessionEvent } from "@alexkroman1/aai/protocol";
+import { withDiscoveredTools } from "@alexkroman1/aai/testing";
+import { errorMessage } from "@alexkroman1/aai/utils";
+import {
+  customEventsIn,
+  lastStateIn,
+  type RunCodeExecutor,
+  toolResultIn,
+} from "@alexkroman1/aai-runtime/eval";
+import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import { z } from "zod";
+import authoredAgent from "./agent.ts";
+import { CATEGORIES, MOODS } from "./shared.ts";
+import systemPrompt from "./system-prompt.md?raw";
+
+/**
+ * The def a DEPLOYED agent runs: authored, plus what `tools/` declares.
+ *
+ * The glob is written HERE rather than reached for from a shared helper because
+ * this file SHIPS — a scaffolded project has no repo helper to import. Without
+ * it the eval would drive an agent with no `recommend` at all, which is the one
+ * failure a green eval must never be able to hide.
+ *
+ * And plus its PROMPT. `agent.ts` does not declare one — `system-prompt.md` is
+ * resolved by the BUILD (`aai build`/`aai deploy`), so the raw default export
+ * carries the FRAMEWORK DEFAULT prompt. An eval that drives it measures a
+ * different agent than the one that deploys, and every tool-choice claim below
+ * then passes or fails for the wrong reason.
+ */
+const agentDef = withSystemPrompt(
+  withDiscoveredTools(authoredAgent, import.meta.glob("./tools/*.ts", { eager: true })),
+  systemPrompt,
+);
+
+/**
+ * One `recommend` answer, and the whole projection, as the wire carries them.
+ *
+ * Schemas rather than casts, which is what `toolResultIn` and `lastStateIn`
+ * take one for: a shelf or a projection that stopped matching FAILS here naming
+ * the field, where a cast hands the assertions `undefined` and fails a line
+ * later on something unrelated.
+ */
+const RecSchema = z.object({
+  category: z.enum(CATEGORIES),
+  mood: z.enum(MOODS),
+  picks: z.array(z.string()),
+});
+const ProjectedNight = z.object({ recs: z.array(RecSchema) });
+
+/**
+ * The night's log as the PAGE has it: the last `syncState` frame pushed.
+ *
+ * This agent declares one projection, so the frame IS `nightProjection`'s
+ * result — the same `{ recs }` value `useAgentState(nightProjection)` reads in
+ * `client.tsx`. Asserting on it is asserting on what the sidebar shows.
+ */
+const pushedRecs = (events: readonly SessionEvent[]) =>
+  lastStateIn(events, ProjectedNight)?.recs ?? [];
+
+/** The `wind_down` nudges in `events` — `customEventsIn` filters by name. */
+const nudges = (events: readonly SessionEvent[]) => customEventsIn(events, "wind_down");
+
+/**
+ * A `run_code` executor, so the sleep-cycle case can assert the ANSWER.
+ *
+ * The builtin refuses without one — the Modal container is the security
+ * boundary, and off-platform there is none — which left this template's
+ * headline feature assertable as a CALL and never as a number. A `node:vm`
+ * context with a capturing `console.log` is what a developer would reach for on
+ * their own machine, and it is enough: the code under test is arithmetic the
+ * model wrote, not a program.
+ *
+ * A template eval imports from `@alexkroman1/aai-runtime/eval` and
+ * `/eval/vitest` and nowhere else — the ROOT barrel drags the host runtime's
+ * node-reaching module graph into this project's TypeScript program, which is
+ * three errors in runtime files no eval ever calls. `RunCodeExecutor` is
+ * re-exported from `/eval` for exactly that reason.
+ */
+const runCode: RunCodeExecutor = async (code) => {
+  const lines: string[] = [];
+  const log = (...args: unknown[]): void => {
+    lines.push(args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" "));
+  };
+  try {
+    runInNewContext(code, { console: { log } }, { timeout: 1000 });
+  } catch (err) {
+    return { error: errorMessage(err) };
+  }
+  return lines.join("\n");
+};
+
+/** Two-digit, for the clock arithmetic below. */
+const pad = (n: number): string => String(n).padStart(2, "0");
+
+/**
+ * Every bedtime `system-prompt.md`'s recipe admits for a given wake-up hour,
+ * in each of the spellings a tutor might print it in.
+ *
+ * Derived from the recipe rather than typed out — 90 minutes a cycle plus 15 to
+ * fall asleep, wrapped into the previous day — so this and the prompt cannot
+ * disagree about the arithmetic the case is checking.
+ */
+const bedtimesFor = (wakeHour: number): string[] =>
+  [3, 4, 5, 6].flatMap((cycles) => {
+    const at = (wakeHour * 60 - (cycles * 90 + 15) + 1440) % 1440;
+    const hour = Math.floor(at / 60);
+    const minute = pad(at % 60);
+    // Three spellings per time, because the prompt asks for HH:MM and a tutor
+    // reasonably prints any of them: padded, unpadded (measured — a live run
+    // printed "0:45" for four cycles), and the 12-hour clock.
+    return [
+      `${pad(hour)}:${minute}`,
+      `${hour}:${minute}`,
+      `${hour % 12 === 0 ? 12 : hour % 12}:${minute}`,
+    ];
+  });
+
+describeEval(
+  agentDef,
+  (test) => {
+    test(
+      "turns a mood into the category and mood the tool takes",
+      async ({ session }) => {
+        const turn = await session.say("I want something cozy to watch tonight.");
+
+        // "to watch" is the category and "cozy" is the mood; the shelf is the
+        // tool's, so answering from the model's own taste is the regression.
+        expect(turn.toolCalls.map((c) => c.name)).toEqual(["recommend"]);
+        const call = turn.toolCalls[0]!;
+        expect(call.args).toEqual({ category: "movie", mood: "cozy" });
+
+        // And what it read out came back from the shelf: the tool answers with
+        // the picks for exactly the pair it was asked for.
+        const rec = toolResultIn(turn.toolCalls, "recommend", RecSchema);
+        expect(rec).toMatchObject({ category: "movie", mood: "cozy" });
+        expect(rec.picks.length).toBeGreaterThan(0);
+        // It read out what the shelf handed back rather than a title of its own.
+        expect(rec.picks.some((pick) => turn.text.includes(pick))).toBe(true);
+      },
+      {
+        stubReply: [
+          { tool: "recommend", args: { category: "movie", mood: "cozy" } },
+          "Paddington 2 is the coziest thing I own.",
+        ],
+      },
+    );
+
+    test(
+      "the night's log keeps what an earlier turn picked, newest first",
+      async ({ session }) => {
+        await session.say("I want something cozy to watch tonight.");
+        const turn = await session.say("Now give me something spooky to read.");
+
+        expect(turn.toolCalls.map((c) => c.name)).toEqual(["recommend"]);
+        expect(turn.toolCalls[0]!.args).toEqual({ category: "book", mood: "spooky" });
+
+        // The slot survived the turn boundary: the frame the page renders after
+        // the second answer still carries the first, and the newest is first —
+        // which is the order the sidebar lists them in.
+        const recs = pushedRecs(session.events());
+        expect(recs.map((r) => `${r.category}/${r.mood}`)).toEqual(["book/spooky", "movie/cozy"]);
+      },
+      {
+        stubReply: [
+          { tool: "recommend", args: { category: "movie", mood: "cozy" } },
+          "Paddington 2 it is.",
+          { tool: "recommend", args: { category: "book", mood: "spooky" } },
+          "Mexican Gothic, then.",
+        ],
+      },
+    );
+
+    test(
+      "the wind-down nudge arrives once, on the third pick",
+      async ({ session }) => {
+        const first = await session.say("I want something cozy to watch tonight.");
+        expect(nudges(first.events)).toEqual([]);
+        const second = await session.say("Now give me something spooky to read.");
+        expect(nudges(second.events)).toEqual([]);
+
+        // Counted in the SLOT, so the third pick is only the third if the two
+        // before it were still there — and it is a `ctx.send`, not a field on the
+        // projection, so it must arrive exactly once and never be replayed.
+        const third = await session.say("And some chill music too.");
+        expect(nudges(third.events)).toHaveLength(1);
+
+        const fourth = await session.say("One funny book as well, please.");
+        expect(nudges(fourth.events)).toEqual([]);
+        expect(nudges(session.events())).toHaveLength(1);
+        expect(pushedRecs(session.events())).toHaveLength(4);
+      },
+      {
+        stubReply: [
+          { tool: "recommend", args: { category: "movie", mood: "cozy" } },
+          "Paddington 2 it is.",
+          { tool: "recommend", args: { category: "book", mood: "spooky" } },
+          "Mexican Gothic, then.",
+          { tool: "recommend", args: { category: "music", mood: "chill" } },
+          "Tycho, Dive.",
+          { tool: "recommend", args: { category: "book", mood: "funny" } },
+          "Good Omens.",
+        ],
+      },
+    );
+    test(
+      "works the bedtime out in CODE, and the number is right",
+      async ({ session }) => {
+        const turn = await session.say(
+          "I need to be up at 7 in the morning. When should I fall asleep?",
+        );
+
+        const ran = turn.toolCalls.filter((c) => c.name === "run_code");
+        expect(
+          ran,
+          `tools called: [${turn.toolCalls.map((c) => c.name).join(", ")}]; said: ${turn.text}`,
+        ).not.toEqual([]);
+        // The recipe is the prompt's, and it is two constants: a 90-minute cycle
+        // plus the 15 minutes it takes to fall asleep. Arithmetic done in the
+        // model's head has neither of them anywhere in the code.
+        const code = ran.map((c) => String(c.args.code ?? "")).join("\n");
+        expect(code).toContain("90");
+        expect(code).toContain("15");
+
+        const output = ran.map((c) => c.result ?? "").join("\n");
+        // The builtin really EXECUTED. With no `runCode` executor this string is
+        // "run_code is only available in the sandboxed runtime", which every
+        // assertion about a CALL sails past — so this template's headline
+        // feature could be checked as a call and never as an answer.
+        expect(output).not.toMatch(/only available in the sandboxed runtime/);
+        // And the answer is a whole number of cycles back from 07:00 with the
+        // quarter hour added. A tutor that dropped the 15 lands on :00 and a
+        // tutor that guessed lands anywhere; both fail here.
+        expect(output, `run_code printed: ${output}`).toMatch(new RegExp(bedtimesFor(7).join("|")));
+      },
+      {
+        stubReply: [
+          {
+            tool: "run_code",
+            args: {
+              code: [
+                "const wake = 7 * 60;",
+                "const at = (wake - (6 * 90 + 15) + 1440) % 1440;",
+                "const two = (n) => String(n).padStart(2, '0');",
+                "console.log(two(Math.floor(at / 60)) + ':' + two(at % 60));",
+              ].join("\n"),
+            },
+          },
+          "Aim for nine forty-five tonight — that's six full cycles before seven.",
+        ],
+      },
+    );
+  },
+  // `runCode` is what makes the case above about an ANSWER rather than a call.
+  { runCode },
+);

package/dist/templates/personal-finance/agent.eval.test.ts

@@ -0,0 +1,168 @@
+// An EVAL: does Penny fetch the rate, compute the split, and refuse to hand out
+// investment advice? Run it with `aai eval`.
+//
+// Penny has two builtins and a rule about what she may not say, so the three
+// live cases below are one per promise the prompt makes: fetch_json for
+// anything that moves (rates, crypto), run_code for anything arithmetic, and a
+// not-financial-advice caveat whenever the subject is an investment.
+//
+// The two harness facts, argued at length in
+// `../code-interpreter/agent.eval.test.ts`: `system-prompt.md` is discovered by
+// the build rather than imported, so an eval has to apply it or it measures an
+// agent with no house rules at all; and `run_code` refuses unless the EVAL
+// supplies an executor, which this suite does — so the arithmetic cases assert
+// the answer as well as the code that was submitted. `fetch_json` is
+// unaffected — it makes a real request, so the currency case really does reach a
+// live rates API.
+import { runInNewContext } from "node:vm";
+import { withSystemPrompt } from "@alexkroman1/aai/manifest";
+import { errorMessage } from "@alexkroman1/aai/utils";
+import type { RunCodeExecutor } from "@alexkroman1/aai-runtime/eval";
+import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import authored from "./agent.ts";
+import systemPrompt from "./system-prompt.md?raw";
+
+const agentDef = withSystemPrompt(authored, systemPrompt);
+
+type Turn = { toolCalls: readonly { name: string; args: Record<string, unknown> }[] };
+
+/** The code every `run_code` call in this turn carried, joined. */
+const codeIn = (turn: Turn) =>
+  turn.toolCalls
+    .filter((c) => c.name === "run_code")
+    .map((c) => String(c.args.code ?? ""))
+    .join("\n");
+
+/** Every URL this turn's `fetch_json` calls asked for. */
+const fetchedUrls = (turn: Turn) =>
+  turn.toolCalls.filter((c) => c.name === "fetch_json").map((c) => String(c.args.url ?? ""));
+
+/**
+ * A `run_code` executor, so these cases can assert the ANSWER.
+ *
+ * The builtin refuses without one — the Modal container is the security
+ * boundary, and off-platform there is none — so a case could assert the CALL and
+ * the code it carried, and never what the code came back with. A `node:vm`
+ * context with a capturing `console.log` is what a developer would reach for on
+ * their own machine, and it is enough here: what runs is arithmetic, not a
+ * program. It is NOT a sandbox and does not pretend to be one; a deployed agent
+ * still gets the refusal.
+ *
+ * A template eval imports from `@alexkroman1/aai-runtime/eval` and
+ * `/eval/vitest` and nowhere else — the ROOT barrel drags the host runtime's
+ * node-reaching module graph into this project's TypeScript program, which is
+ * three errors in runtime files no eval ever calls. `RunCodeExecutor` is
+ * re-exported from `/eval` for exactly that reason.
+ */
+const runCode: RunCodeExecutor = async (code) => {
+  const lines: string[] = [];
+  const log = (...args: unknown[]): void => {
+    lines.push(args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" "));
+  };
+  try {
+    runInNewContext(code, { console: { log } }, { timeout: 1000 });
+  } catch (err) {
+    return { error: errorMessage(err) };
+  }
+  return lines.join("\n");
+};
+
+/** What every `run_code` call in this turn PRINTED, joined. */
+const outputIn = (turn: { toolCalls: readonly { name: string; result?: string }[] }) =>
+  turn.toolCalls
+    .filter((c) => c.name === "run_code")
+    .map((c) => c.result ?? "")
+    .join("\n");
+
+describeEval(
+  agentDef,
+  (test) => {
+    test(
+      "splits a bill in code, tip included",
+      async ({ session }) => {
+        const turn = await session.say(
+          "Help me split a 120 dollar bill four ways with a 20 percent tip.",
+        );
+
+        // Three numbers, two operations and a rounding rule: the exact shape of
+        // question a model answers plausibly and wrongly. All three inputs have
+        // to reach the code, or something was worked out in the model's head.
+        expect(turn.toolCalls.map((c) => c.name)).toContain("run_code");
+        const code = codeIn(turn);
+        expect(code).toContain("120");
+        expect(code).toMatch(/\b4\b/);
+        expect(code).toMatch(/20|0\.2/);
+
+        // And the sum came out right: $120 plus 20% is $144, four ways is $36.
+        // Without an executor `run_code` answers with a refusal, so every claim
+        // above is satisfied by an agent that then divides in its head — which is
+        // the failure this template's whole run_code rule exists to prevent.
+        const output = outputIn(turn);
+        expect(output, `run_code printed: ${output}`).toMatch(/\b36(\.0+)?\b/);
+      },
+      { live: true },
+    );
+
+    test(
+      "looks a currency rate up instead of quoting one from memory",
+      async ({ session }) => {
+        const turn = await session.say("What's 100 US dollars in euros right now?");
+
+        // "Right now" is the point. A rate the model remembers is months stale
+        // and has no source, and the prompt names the endpoint to use — so the
+        // regression this catches is Penny answering confidently with no request
+        // at all.
+        const urls = fetchedUrls(turn);
+        expect(urls.length).toBeGreaterThan(0);
+        expect(urls.join(" ")).toMatch(/^https:\/\//);
+        expect(urls.join(" ")).toMatch(/er-api|exchangerate|currency|rates/i);
+      },
+      { live: true },
+    );
+
+    test(
+      "will not tell you to put your savings into crypto",
+      async ({ session }) => {
+        const turn = await session.say("Should I put all my savings into bitcoin?");
+
+        // The one thing this agent must never do straight. The prompt promises a
+        // fluctuation / not-financial-advice caveat whenever the subject is an
+        // investment, and a prompt edit that drops it leaves an agent cheerfully
+        // recommending an all-in bet on a voice call.
+        //
+        // `risk` is in the alternation because the caveat is a CLAIM and this is
+        // a live model's wording: a passing run answered "it's high risk and
+        // prices change fast", which is the promise kept in words none of the
+        // other four alternatives match. A reply that recommends the bet carries
+        // none of the five.
+        expect(turn.text).toMatch(/not (financial|investment) advice|fluctuat|volatil|swing|risk/i);
+        expect(turn.completed).toBe(true);
+      },
+      { live: true },
+    );
+
+    test(
+      "the run_code builtin is wired to the agent's tool executor",
+      async ({ session }) => {
+        const turn = await session.say("What's 20 percent of 120 dollars?");
+
+        // A tool the agent does not declare produces a `tool.called` with no
+        // result, so the paired result is what says `builtinTools` still resolves
+        // to something executable. Scripted deliberately on run_code rather than
+        // fetch_json: a scripted tool call really runs, and a wiring check should
+        // not depend on somebody else's API being up. The ANSWER rather than
+        // `toBeDefined()`, which the refusal string satisfied too.
+        const [call] = turn.toolCalls;
+        expect(call?.name).toBe("run_code");
+        expect(call?.result).toBe("24");
+        expect(turn.completed).toBe(true);
+      },
+      {
+        stubReply: [{ tool: "run_code", args: { code: "console.log(120 * 0.2)" } }, "That's $24."],
+      },
+    );
+  },
+  // `runCode` is what makes these cases about the ANSWER and not just the call.
+  { runCode },
+);

package/dist/templates/personal-finance/system-prompt.md

@@ -1,6 +1,10 @@
 You are Penny, a friendly personal finance assistant. You help people with currency conversions, cryptocurrency prices, loan calculations, savings projections, and splitting bills.
 
 Rules:
+- Never do arithmetic in your head. EVERY figure you speak — a tip, a split, a
+  payment, a projection, a converted amount — comes out of run_code, including
+  ones you could do instantly. A number you worked out yourself is a number you
+  invented, and the caller spends money on it.
 - Always show your math clearly when explaining calculations
 - When discussing investments or crypto, remind users that prices fluctuate and this is not financial advice
 - Be encouraging about savings goals

package/dist/templates/pipeline-simple/agent.eval.test.ts

@@ -0,0 +1,78 @@
+// An EVAL: does the swapped-in stage actually answer? Run it with `aai eval`.
+//
+// `agent.test.ts` asserts the DESCRIPTOR — `llm.kind === "anthropic"`, the two
+// unset stages filling to AssemblyAI in the deployable config. It runs no agent,
+// so it cannot tell a working provider from a model id that was retired last
+// month: `toAgentConfig` is happy either way. This file is the other half. A
+// live run here opens a real Anthropic connection with the real model string,
+// which is the one claim this template makes and the one that rots on its own.
+//
+// **A live run needs ANTHROPIC_API_KEY, not the AssemblyAI key.** That is the
+// template's own lesson arriving in the test suite: swap a stage and you bring
+// that stage's credential. Without it `describeEval` announces SCRIPTED and the
+// live-only case below is skipped — which is a wiring check, not a measurement,
+// and the banner says so on every run.
+import { describeEval } from "@alexkroman1/aai-runtime/eval/vitest";
+import { expect } from "vitest";
+import agentDef from "./agent.ts";
+
+/** Roughly how many words a reply is, for the spoken-length claim. */
+const wordCount = (text: string): number => text.trim().split(/\s+/).filter(Boolean).length;
+
+describeEval(agentDef, (test) => {
+  test(
+    "answers on the declared model, without reaching for a tool",
+    async ({ session }) => {
+      // `say()` hands back THAT turn, so this is a claim about the reply to
+      // this question rather than about everything said so far — which already
+      // includes the greeting.
+      const turn = await session.say("What is the capital of France?");
+
+      expect(turn.completed).toBe(true);
+      expect(turn.text).toMatch(/paris/i);
+      // This agent declares no tools and no builtins, so a tool call here
+      // would mean something got added by accident.
+      expect(turn.toolCalls).toEqual([]);
+      expect(turn.events.some((e) => e.type === "error.reported")).toBe(false);
+    },
+    { stubReply: "Paris is the capital of France." },
+  );
+
+  test(
+    "keeps the thread across two turns",
+    async ({ session }) => {
+      await session.say("My name is Sam and I work in Berlin.");
+      const turn = await session.say("Which city did I say I work in?");
+
+      expect(turn.text).toMatch(/berlin/i);
+      expect(turn.events.some((e) => e.type === "error.reported")).toBe(false);
+    },
+    // One scripted reply per turn: the second is the one under test, and a
+    // script that answered only the first would fail the case it is meant to
+    // let run. Scripted, the claim is that the session drives two turns; live,
+    // it is that the model still has the first one.
+    { stubReply: ["Good to meet you, Sam.", "You said Berlin."] },
+  );
+
+  test(
+    "keeps a reply speakable, even when the question invites an essay",
+    async ({ session }) => {
+      const turn = await session.say(
+        "Tell me everything you know about the history of the Roman Empire.",
+      );
+
+      // This template ships no prompt of its own, so what holds the reply down
+      // is `DEFAULT_SYSTEM_PROMPT`'s SPEAKING section — two sentences, about
+      // thirty spoken words, no markdown. That is a measured rule (interruption
+      // rate climbs from 17% under ten words to 59% past thirty-five), and a
+      // stage swap that quietly loses it produces an agent nobody can hold a
+      // call with. The ceiling is generous against the rule's own thirty so the
+      // case fails on an essay rather than on a long sentence.
+      expect(wordCount(turn.text)).toBeLessThanOrEqual(80);
+      expect(turn.text).not.toMatch(/[*#`]|^\s*[-•]\s/m);
+    },
+    // Live only: a scripted reply's length is this file's own choice, so
+    // asserting it in stub mode would measure nothing.
+    { live: true },
+  );
+});